pi-advisor-flow 0.5.5 → 0.6.0

package/src/scout-curation.ts

@@ -0,0 +1,84 @@
+import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
+import { advisorScoutEnabledRef, executorRef } from "./config/state.ts";
+import {
+  runAdvisorScout,
+  type ScoutLifecycleEvent,
+  type ScoutOutcome,
+} from "./scout.ts";
+import {
+  buildScoutManifest,
+  reconstructScoutConversation,
+  SCOUT_MANIFEST_MAX_BYTES,
+} from "./scout-context.ts";
+
+/** Runs Scout over the manifested branch and reconstructs the curated
+ * conversation, falling back to the exact legacy conversation on any
+ * non-cancellation Scout failure. */
+export const curateAdvisorConversation = async (
+  ctx: ExtensionContext,
+  legacyConversation: string,
+  signal?: AbortSignal,
+  onScout?: (event: ScoutLifecycleEvent) => void,
+  enabled = advisorScoutEnabledRef,
+  runScout: typeof runAdvisorScout = runAdvisorScout,
+  currentInvocationId?: string,
+  maxChars?: number
+): Promise<{
+  conversation: string;
+  scout?: Exclude<ScoutOutcome, { cancelled: true }>;
+}> => {
+  if (!enabled) {
+    return { conversation: legacyConversation };
+  }
+  if (maxChars !== undefined && maxChars <= 0) {
+    return { conversation: "" };
+  }
+  const built = buildScoutManifest(ctx, {
+    currentInvocationId,
+    maxConversationChars: maxChars,
+    maxManifestBytes: SCOUT_MANIFEST_MAX_BYTES,
+  });
+  if (!built.ok) {
+    const scout: Exclude<ScoutOutcome, { cancelled: true }> = {
+      category: built.reason,
+      message: built.message,
+      metrics: {
+        availableCount: 0,
+        inputBytes: 0,
+        latencyMs: 0,
+        omittedBeforeScout: 0,
+        selectedCount: 0,
+      },
+      model: executorRef,
+      ok: false,
+    };
+    onScout?.({ outcome: scout, type: "fallback" });
+    return { conversation: legacyConversation, scout };
+  }
+  const outcome = await runScout(
+    ctx,
+    built.manifest,
+    signal,
+    onScout,
+    undefined,
+    undefined
+  );
+  if (!outcome.ok && outcome.cancelled) {
+    throw signal?.reason instanceof Error
+      ? signal.reason
+      : new Error("Advisor operation cancelled during Scout.");
+  }
+  let conversation = legacyConversation;
+  if (outcome.ok) {
+    conversation =
+      maxChars === undefined
+        ? outcome.conversation
+        : reconstructScoutConversation(
+            built.manifest,
+            outcome.selection.selectedIds,
+            outcome.selection.synthesis,
+            maxChars
+          );
+  }
+  return { conversation, scout: outcome };
+};

package/src/scout-groups.ts

@@ -0,0 +1,485 @@
+import { createHash } from "node:crypto";
+import type { AdvisorToolPolicies } from "./config/types.ts";
+import {
+  byteLength,
+  contentParts,
+  isRecord,
+  type RecordValue,
+} from "./content-utils.ts";
+import { conversationEntry } from "./conversation.ts";
+import { invalid, toolCallId, toolCalls } from "./scout-protocol.ts";
+import {
+  type InvalidProtocolResult,
+  SCOUT_LABEL_MAX_CHARS,
+  type ScoutContextGroup,
+  type ScoutGroupKind,
+} from "./scout-types.ts";
+
+export interface GroupPass {
+  groups: ScoutContextGroup[];
+  ok: true;
+  protocolOmittedBytes: number;
+  protocolOmittedCount: number;
+}
+
+const SPEAKER_PREFIX = /^(User|Executor):\s*/;
+
+const boundedLabel = (value: string) =>
+  [...value.replace(/\s+/g, " ").trim()]
+    .slice(0, SCOUT_LABEL_MAX_CHARS)
+    .join("");
+
+const labelFor = (kind: ScoutGroupKind, content: string) => {
+  const preview = boundedLabel(content.replace(SPEAKER_PREFIX, ""));
+  const prefix: Record<ScoutGroupKind, string> = {
+    assistant: "Executor",
+    compaction: "Compaction summary",
+    "pending-invocation": "Current Advisor invocation",
+    "tool-exchange": "Tool exchange",
+    user: "User request",
+  };
+  return boundedLabel(`${prefix[kind]}: ${preview || "(no text)"}`);
+};
+
+const stableId = (
+  index: number,
+  entryIds: string[],
+  kind: ScoutGroupKind,
+  content: string
+) =>
+  `g_${createHash("sha256")
+    .update(JSON.stringify([index, entryIds, kind, content]))
+    .digest("hex")
+    .slice(0, 16)}`;
+
+/** The wire projection shared by manifest transport and size accounting. */
+export const groupWire = (group: ScoutContextGroup) => ({
+  bytes: group.bytes,
+  content: group.content,
+  id: group.id,
+  kind: group.kind,
+  label: group.label,
+  required: group.required,
+});
+
+export const groupWireBytes = (group: ScoutContextGroup) =>
+  byteLength(JSON.stringify(groupWire(group)));
+
+const createGroup = (
+  originalIndex: number,
+  entryIds: string[],
+  kind: ScoutGroupKind,
+  content: string,
+  required: boolean
+): ScoutContextGroup => ({
+  bytes: byteLength(content),
+  content,
+  id: stableId(originalIndex, entryIds, kind, content),
+  kind,
+  label: labelFor(kind, content),
+  originalIndex,
+  required,
+});
+
+const pendingAdvisorArguments = (value: unknown): RecordValue => {
+  if (!isRecord(value)) {
+    return {};
+  }
+  const allowed: RecordValue = {};
+  for (const key of ["gitContext", "question"]) {
+    if (key in value) {
+      allowed[key] = value[key];
+    }
+  }
+  return allowed;
+};
+
+const pendingInvocationDisclosure = (
+  entry: RecordValue,
+  invocationId: string,
+  toolResultMaxLines: number,
+  toolResultMaxBytes: number,
+  policies: AdvisorToolPolicies,
+  redact: boolean,
+  disclosed: string
+) => {
+  if (!isRecord(entry.message)) {
+    return disclosed;
+  }
+  const content = contentParts(entry.message.content).map((part) => {
+    if (
+      !isRecord(part) ||
+      part.type !== "toolCall" ||
+      toolCallId(part) !== invocationId ||
+      part.name !== "ask_advisor"
+    ) {
+      return part;
+    }
+    return { ...part, arguments: pendingAdvisorArguments(part.arguments) };
+  });
+  return conversationEntry(
+    { ...entry, message: { ...entry.message, content } },
+    toolResultMaxLines,
+    toolResultMaxBytes,
+    policies,
+    redact
+  );
+};
+
+export interface DisclosureCaps {
+  currentInvocationId?: string;
+  policies: AdvisorToolPolicies;
+  redact: boolean;
+  toolResultMaxBytes: number;
+  toolResultMaxLines: number;
+}
+
+type ToolExchangeOutcome =
+  | { kind: "group"; group: ScoutContextGroup }
+  | { kind: "omitted"; bytes: number }
+  | { kind: "invalid"; message: string };
+
+/** Groups an assistant message with its tool calls and matched results into
+ * one indivisible tool-exchange (or pending-invocation) group. */
+const toolExchangeGroup = (
+  entry: RecordValue,
+  index: number,
+  entryId: string,
+  disclosed: string,
+  immediate: unknown,
+  indexed: {
+    callOwners: Map<string, { index: number; name: string }>;
+    resultsByCall: Map<string, Array<{ entry: RecordValue; index: number }>>;
+  },
+  consumedResultIndexes: Set<number>,
+  caps: DisclosureCaps
+): ToolExchangeOutcome => {
+  const calls = toolCalls(entry.message as RecordValue);
+  const callIds = calls.map(toolCallId) as string[];
+  const adjacentFailure = adjacentResultMismatch(
+    immediate,
+    index,
+    callIds,
+    indexed.callOwners
+  );
+  if (adjacentFailure) {
+    return { kind: "invalid", message: adjacentFailure };
+  }
+  const missing = new Set<string>();
+  const resultParts: string[] = [];
+  const resultEntryIds: string[] = [];
+  const matchFailure = collectResults(
+    calls,
+    callIds,
+    index,
+    indexed.resultsByCall,
+    consumedResultIndexes,
+    caps,
+    missing,
+    resultParts,
+    resultEntryIds
+  );
+  if (matchFailure) {
+    return { kind: "invalid", message: matchFailure };
+  }
+  if (missing.size > 0) {
+    return missingOutcome(
+      entry,
+      index,
+      entryId,
+      disclosed,
+      callIds,
+      missing,
+      resultParts,
+      resultEntryIds,
+      caps
+    );
+  }
+  return {
+    group: createGroup(
+      index,
+      [entryId, ...resultEntryIds],
+      "tool-exchange",
+      [disclosed, ...resultParts].join("\n\n"),
+      false
+    ),
+    kind: "group",
+  };
+};
+
+const adjacentResultMismatch = (
+  immediate: unknown,
+  index: number,
+  callIds: string[],
+  callOwners: Map<string, { index: number; name: string }>
+): string | undefined => {
+  const next = immediate as RecordValue | undefined;
+  if (
+    next?.type === "message" &&
+    isRecord(next.message) &&
+    next.message.role === "toolResult" &&
+    typeof next.message.toolCallId === "string" &&
+    !callIds.includes(next.message.toolCallId) &&
+    !callOwners.has(next.message.toolCallId)
+  ) {
+    return `Tool result at context entry ${index + 1} does not match its adjacent assistant group.`;
+  }
+  return undefined;
+};
+
+const collectResults = (
+  calls: RecordValue[],
+  callIds: string[],
+  index: number,
+  resultsByCall: Map<string, Array<{ entry: RecordValue; index: number }>>,
+  consumedResultIndexes: Set<number>,
+  caps: DisclosureCaps,
+  missing: Set<string>,
+  resultParts: string[],
+  resultEntryIds: string[]
+): string | undefined => {
+  for (const [callIndex, callId] of callIds.entries()) {
+    const resultMatch = resultsByCall.get(callId)?.[0];
+    if (!resultMatch) {
+      missing.add(callId);
+      continue;
+    }
+    if (resultMatch.index <= index) {
+      return `Tool result at context entry ${resultMatch.index} precedes its assistant call.`;
+    }
+    const resultMessage = resultMatch.entry.message;
+    const expectedName =
+      typeof calls[callIndex].name === "string"
+        ? calls[callIndex].name
+        : "unknown";
+    if (!isRecord(resultMessage) || resultMessage.toolName !== expectedName) {
+      return `Tool result at context entry ${resultMatch.index} conflicts with call ${callId}.`;
+    }
+    consumedResultIndexes.add(resultMatch.index);
+    const resultText = conversationEntry(
+      resultMatch.entry,
+      caps.toolResultMaxLines,
+      caps.toolResultMaxBytes,
+      caps.policies,
+      caps.redact
+    );
+    if (resultText) {
+      resultParts.push(resultText);
+    }
+    resultEntryIds.push(
+      typeof resultMatch.entry.id === "string"
+        ? resultMatch.entry.id
+        : String(resultMatch.index)
+    );
+  }
+  return undefined;
+};
+
+const missingOutcome = (
+  entry: RecordValue,
+  index: number,
+  entryId: string,
+  disclosed: string,
+  callIds: string[],
+  missing: Set<string>,
+  resultParts: string[],
+  resultEntryIds: string[],
+  caps: DisclosureCaps
+): ToolExchangeOutcome => {
+  const { currentInvocationId } = caps;
+  const pendingCurrentInvocation =
+    currentInvocationId !== undefined &&
+    missing.size === 1 &&
+    missing.has(currentInvocationId) &&
+    callIds.includes(currentInvocationId);
+  if (!pendingCurrentInvocation) {
+    return {
+      bytes: byteLength([disclosed, ...resultParts].join("\n\n")),
+      kind: "omitted",
+    };
+  }
+  const pendingDisclosed = pendingInvocationDisclosure(
+    entry,
+    currentInvocationId,
+    caps.toolResultMaxLines,
+    caps.toolResultMaxBytes,
+    caps.policies,
+    caps.redact,
+    disclosed
+  );
+  return {
+    group: createGroup(
+      index,
+      [entryId, ...resultEntryIds],
+      "pending-invocation",
+      [pendingDisclosed ?? disclosed, ...resultParts].join("\n\n"),
+      true
+    ),
+    kind: "group",
+  };
+};
+
+/** Second entry pass: groups each disclosed entry, delegating
+ * assistant-with-calls entries to the tool-exchange builder. */
+export const buildGroups = (
+  entries: unknown[],
+  indexed: {
+    callOwners: Map<string, { index: number; name: string }>;
+    latestUserIndex: number;
+    resultsByCall: Map<string, Array<{ entry: RecordValue; index: number }>>;
+  },
+  caps: DisclosureCaps & {
+    maxGroupBytes: number;
+  }
+): GroupPass | InvalidProtocolResult => {
+  const groups: ScoutContextGroup[] = [];
+  const consumedResultIndexes = new Set<number>();
+  let protocolOmittedBytes = 0;
+  let protocolOmittedCount = 0;
+  for (let index = 0; index < entries.length; index += 1) {
+    const entry = entries[index] as unknown as RecordValue;
+    const disclosed = conversationEntry(
+      entry,
+      caps.toolResultMaxLines,
+      caps.toolResultMaxBytes,
+      caps.policies,
+      caps.redact
+    );
+    if (!disclosed) {
+      continue;
+    }
+    const outcome = groupForEntry(
+      entry,
+      index,
+      disclosed,
+      entries[index + 1],
+      indexed,
+      consumedResultIndexes,
+      caps
+    );
+    if (outcome.kind === "invalid") {
+      return invalid(outcome.message);
+    }
+    if (outcome.kind === "omitted") {
+      protocolOmittedCount += 1;
+      protocolOmittedBytes += outcome.bytes;
+    } else if (outcome.kind === "group") {
+      groups.push(outcome.group);
+    }
+  }
+  return {
+    groups,
+    ok: true,
+    protocolOmittedBytes,
+    protocolOmittedCount,
+  };
+};
+
+type EntryOutcome =
+  | { kind: "group"; group: ScoutContextGroup }
+  | { kind: "omitted"; bytes: number }
+  | { kind: "invalid"; message: string }
+  | { kind: "skipped" };
+
+const groupForEntry = (
+  entry: RecordValue,
+  index: number,
+  disclosed: string,
+  immediate: unknown,
+  indexed: {
+    callOwners: Map<string, { index: number; name: string }>;
+    latestUserIndex: number;
+    resultsByCall: Map<string, Array<{ entry: RecordValue; index: number }>>;
+  },
+  consumedResultIndexes: Set<number>,
+  caps: DisclosureCaps
+): EntryOutcome => {
+  const entryId = typeof entry.id === "string" ? entry.id : String(index);
+  if (entry.type !== "message" || !isRecord(entry.message)) {
+    return {
+      group: createGroup(index, [entryId], "compaction", disclosed, false),
+      kind: "group",
+    };
+  }
+  const { message } = entry;
+  if (message.role === "user") {
+    return {
+      group: createGroup(
+        index,
+        [entryId],
+        "user",
+        disclosed,
+        index === indexed.latestUserIndex
+      ),
+      kind: "group",
+    };
+  }
+  if (message.role === "toolResult") {
+    return toolResultOutcome(
+      message,
+      index,
+      disclosed,
+      indexed.callOwners,
+      consumedResultIndexes
+    );
+  }
+  if (message.role !== "assistant") {
+    return { kind: "skipped" };
+  }
+  if (!hasCalls(message)) {
+    return {
+      group: createGroup(index, [entryId], "assistant", disclosed, false),
+      kind: "group",
+    };
+  }
+  return toolExchangeGroup(
+    entry,
+    index,
+    entryId,
+    disclosed,
+    immediate,
+    indexed,
+    consumedResultIndexes,
+    caps
+  );
+};
+
+const toolResultOutcome = (
+  message: RecordValue,
+  index: number,
+  disclosed: string,
+  callOwners: Map<string, { index: number; name: string }>,
+  consumedResultIndexes: Set<number>
+): EntryOutcome => {
+  if (consumedResultIndexes.has(index)) {
+    return { kind: "skipped" };
+  }
+  if (ownerOf(message, callOwners)) {
+    return {
+      kind: "invalid",
+      message: `Tool result at context entry ${index} precedes or conflicts with its retained call.`,
+    };
+  }
+  // Retained results without their call are unavailable optional evidence
+  // and are never offered to Scout.
+  return { bytes: byteLength(disclosed), kind: "omitted" };
+};
+
+const ownerOf = (
+  message: RecordValue,
+  callOwners: Map<string, { index: number; name: string }>
+) =>
+  typeof message.toolCallId === "string"
+    ? callOwners.get(message.toolCallId)
+    : undefined;
+
+const hasCalls = (message: RecordValue) =>
+  contentPartsOf(message).some(
+    (part) => isRecord(part) && part.type === "toolCall"
+  );
+
+const contentPartsOf = (message: RecordValue) =>
+  Array.isArray(message.content) ? message.content : [];
+
+/** Final pass: selects groups within the transport and conversation budgets,
+ * keeping every required group. */

package/src/scout-protocol.ts

@@ -0,0 +1,122 @@
+import { contentParts, isRecord, type RecordValue } from "./content-utils.ts";
+import { textFrom } from "./conversation.ts";
+import type { InvalidProtocolResult } from "./scout-types.ts";
+
+/** The single invalid-protocol failure constructor. */
+export const invalid = (message: string): InvalidProtocolResult => ({
+  message,
+  ok: false,
+  reason: "invalid-protocol",
+});
+
+export interface ToolCallIndex {
+  callOwners: Map<string, { index: number; name: string }>;
+  latestUserIndex: number;
+  resultsByCall: Map<string, Array<{ entry: RecordValue; index: number }>>;
+}
+
+export const toolCalls = (message: RecordValue) =>
+  contentParts(message.content).filter(
+    (part): part is RecordValue => isRecord(part) && part.type === "toolCall"
+  );
+
+export const toolCallId = (part: RecordValue) =>
+  typeof part.id === "string" ? part.id : undefined;
+
+/** First entry pass: indexes assistant tool calls and their results, tracking
+ * the latest user message. Returns an invalid result on protocol violations. */
+export const indexToolCalls = (
+  entries: unknown[]
+): { index: ToolCallIndex; ok: true } | InvalidProtocolResult => {
+  const callOwners = new Map<string, { index: number; name: string }>();
+  const resultsByCall = new Map<
+    string,
+    Array<{ entry: RecordValue; index: number }>
+  >();
+  const state = { latestUserIndex: -1 };
+  for (let index = 0; index < entries.length; index += 1) {
+    const failure = indexEntry(
+      entries[index] as unknown as RecordValue,
+      index,
+      callOwners,
+      resultsByCall,
+      state
+    );
+    if (failure) {
+      return failure;
+    }
+  }
+  for (const [id, results] of resultsByCall) {
+    if (results.length > 1) {
+      return invalid(`Tool call ${id} has duplicate result messages.`);
+    }
+  }
+  return {
+    index: {
+      callOwners,
+      latestUserIndex: state.latestUserIndex,
+      resultsByCall,
+    },
+    ok: true,
+  };
+};
+
+const indexEntry = (
+  entry: RecordValue,
+  index: number,
+  callOwners: Map<string, { index: number; name: string }>,
+  resultsByCall: Map<string, Array<{ entry: RecordValue; index: number }>>,
+  state: { latestUserIndex: number }
+): InvalidProtocolResult | undefined => {
+  if (entry.type !== "message" || !isRecord(entry.message)) {
+    return undefined;
+  }
+  if (entry.message.role === "user" && textFrom(entry.message.content)) {
+    state.latestUserIndex = index;
+  }
+  if (entry.message.role === "assistant") {
+    return indexAssistantCalls(entry.message, index, callOwners);
+  }
+  if (entry.message.role === "toolResult") {
+    return indexToolResult(entry, entry.message, index, resultsByCall);
+  }
+  return undefined;
+};
+
+const indexToolResult = (
+  entry: RecordValue,
+  message: RecordValue,
+  index: number,
+  resultsByCall: Map<string, Array<{ entry: RecordValue; index: number }>>
+): InvalidProtocolResult | undefined => {
+  const id = message.toolCallId;
+  if (typeof id !== "string") {
+    return invalid(
+      `Tool result at context entry ${index} has no tool-call ID.`
+    );
+  }
+  const results = resultsByCall.get(id) ?? [];
+  results.push({ entry, index });
+  resultsByCall.set(id, results);
+  return undefined;
+};
+
+const indexAssistantCalls = (
+  message: RecordValue,
+  index: number,
+  callOwners: Map<string, { index: number; name: string }>
+): InvalidProtocolResult | undefined => {
+  for (const call of toolCalls(message)) {
+    const id = toolCallId(call);
+    if (!id || callOwners.has(id)) {
+      return invalid(
+        `Assistant tool calls at context entry ${index} have missing or duplicate IDs.`
+      );
+    }
+    callOwners.set(id, {
+      index,
+      name: typeof call.name === "string" ? call.name : "unknown",
+    });
+  }
+  return undefined;
+};