pi-advisor-flow 0.9.2-dev.4 → 0.9.2-dev.5

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-advisor-flow",
-  "version": "0.9.2-dev.4",
+  "version": "0.9.2-dev.5",
   "description": "Advanced Executor/Advisor flow for Pi, fully configurable and extendable.",
   "keywords": [
     "advisor",

package/src/commands/manual-consultation.ts

@@ -15,6 +15,8 @@ import type { CommandRuntime, ManualAdvisorProgressState } from "./types.ts";
 
 interface ManualResultDetails {
   advisor: string;
+  agentRulesBytes?: number;
+  preferenceBytes?: number;
   question?: string;
   text: string;
   usage?: AdvisorUsageSnapshot;
@@ -48,34 +50,35 @@ export const startManualConsultation = async (
   const finishHerdrActivity = runtime.herdrActivity.start();
   let scoutDetails: ScoutToolDetails | undefined;
   try {
-    const { adviceId, markdown, usage } = await runtime.requestAdvisor(
-      ctx,
-      question,
-      controller.signal,
-      (thinking, text) => {
-        if (controller.signal.aborted) {
-          return;
-        }
-        progress.phase = "active";
-        progress.thinking = thinking;
-        progress.text = text;
-        runtime.requestManualRender(ctx);
-      },
-      (event) => {
-        if (!controller.signal.aborted) {
-          runtime.scoutStatus.update(ctx, scoutStatusToken, event);
-          scoutDetails = appendScoutLifecycleEntry(
-            runtime.pi,
-            event,
-            scoutDetails
-          );
-          progress.scout = scoutDetails;
+    const { adviceId, agentRulesBytes, markdown, preferenceBytes, usage } =
+      await runtime.requestAdvisor(
+        ctx,
+        question,
+        controller.signal,
+        (thinking, text) => {
+          if (controller.signal.aborted) {
+            return;
+          }
           progress.phase = "active";
+          progress.thinking = thinking;
+          progress.text = text;
           runtime.requestManualRender(ctx);
-        }
-      },
-      gitContext
-    );
+        },
+        (event) => {
+          if (!controller.signal.aborted) {
+            runtime.scoutStatus.update(ctx, scoutStatusToken, event);
+            scoutDetails = appendScoutLifecycleEntry(
+              runtime.pi,
+              event,
+              scoutDetails
+            );
+            progress.scout = scoutDetails;
+            progress.phase = "active";
+            runtime.requestManualRender(ctx);
+          }
+        },
+        gitContext
+      );
     if (controller.signal.aborted) {
       return;
     }
@@ -103,6 +106,8 @@ export const startManualConsultation = async (
     runtime.updateAdvisorUsageStatus(ctx);
     const details: ManualResultDetails = {
       advisor: advisorRef,
+      agentRulesBytes,
+      preferenceBytes,
       question,
       text: markdown,
     };

package/src/commands/outcome-stats.ts

@@ -0,0 +1,98 @@
+import { aggregateOutcomeRecords, readOutcomeLog } from "../outcome-stats.ts";
+import type { OutcomeStats } from "../outcome-stats.ts";
+import { OUTCOME_LOG_MAX_BYTES } from "../outcomes.ts";
+import type { CommandRuntime } from "./types.ts";
+
+export interface AdvisorStatsDetails {
+  error?: string;
+  malformedLines: number;
+  stats?: OutcomeStats;
+}
+
+const percent = (rate: number | undefined) =>
+  rate === undefined ? "n/a" : `${(rate * 100).toFixed(1)}%`;
+
+const validatedCount = (value: { failed: number; passed: number }) =>
+  value.failed + value.passed;
+
+const timestamp = (value: string | undefined) =>
+  value ? new Date(value).toLocaleString() : "n/a";
+
+export const formatAdvisorStats = ({
+  error,
+  malformedLines,
+  stats,
+}: AdvisorStatsDetails): string => {
+  if (error) {
+    return `**Could not read Advisor outcomes.**\n\n${error}`;
+  }
+  if (!stats || stats.total === 0) {
+    const ignored = malformedLines
+      ? `\n\nIgnored ${malformedLines} malformed ledger line${malformedLines === 1 ? "" : "s"}.`
+      : "";
+    return `**No Advisor outcomes recorded yet.**\n\nEnable outcome logging and record an outcome after an Advisor response to build this report.${ignored}`;
+  }
+  const { validationByAdoption } = stats;
+  const { followed } = validationByAdoption;
+  const notFollowed = validationByAdoption["not-followed"];
+  const triggerLines = Object.entries(stats.byTrigger).map(
+    ([trigger, count]) => `- ${trigger}: ${count}`
+  );
+  const adoptionLines = Object.entries(stats.adoption).map(
+    ([adoption, count]) => `- ${adoption}: ${count}`
+  );
+  const validationLines = [
+    `- followed: ${percent(followed.passRate)} passed (${followed.passed}/${validatedCount(followed)})`,
+    `- not-followed: ${percent(notFollowed.passRate)} passed (${notFollowed.passed}/${validatedCount(notFollowed)})`,
+  ];
+  const ignored = malformedLines
+    ? `\n\nIgnored ${malformedLines} malformed ledger line${malformedLines === 1 ? "" : "s"}.`
+    : "";
+  return [
+    `**${stats.total} Advisor outcome${stats.total === 1 ? "" : "s"}**`,
+    "",
+    "**By trigger**",
+    ...triggerLines,
+    "",
+    "**Adoption**",
+    ...adoptionLines,
+    "",
+    "**Validation passed rate (passed / passed + failed)**",
+    ...validationLines,
+    "",
+    `Distinct advices seen: ${stats.distinctAdvices}`,
+    `Time window: ${timestamp(stats.firstTimestamp)} → ${timestamp(stats.lastTimestamp)}`,
+    `Ledger window: capped at ${OUTCOME_LOG_MAX_BYTES / (1024 * 1024)} MiB and rewritten on overflow; this report covers only retained records.${ignored}`,
+  ].join("\n");
+};
+
+const readAdvisorStats = async (): Promise<AdvisorStatsDetails> => {
+  try {
+    const log = await readOutcomeLog();
+    const stats = aggregateOutcomeRecords(log.records, log.malformedLines);
+    return { malformedLines: log.malformedLines, stats };
+  } catch (error) {
+    return {
+      error: error instanceof Error ? error.message : String(error),
+      malformedLines: 0,
+    };
+  }
+};
+
+export const registerOutcomeStatsCommand = (runtime: CommandRuntime) => {
+  runtime.pi.registerCommand("advisor-stats", {
+    description: "Show retained Advisor outcome adoption and validation stats",
+    handler: async (_args, _ctx) => {
+      const details = await readAdvisorStats();
+      runtime.pi.sendMessage(
+        {
+          content: [],
+          customType: "advisor-stats-result",
+          details,
+          display: true,
+        },
+        { deliverAs: "steer", triggerTurn: false }
+      );
+    },
+  });
+};

package/src/commands/registration.ts

@@ -7,6 +7,7 @@ import { activateAdvisor } from "./activation.ts";
 import { registerCommandLifecycle } from "./lifecycle.ts";
 import { registerManualCommand } from "./manual-command.ts";
 import { registerModelCommands } from "./model-commands.ts";
+import { registerOutcomeStatsCommand } from "./outcome-stats.ts";
 import { registerCommandRenderers } from "./renderers.ts";
 import { createCommandRuntime } from "./runtime.ts";
 import { registerSettingsCommands } from "./settings-commands.ts";
@@ -24,5 +25,6 @@ export const registerCommands = (
   registerCommandLifecycle(runtime, activate);
   registerManualCommand(runtime);
   registerModelCommands(runtime);
+  registerOutcomeStatsCommand(runtime);
   registerSettingsCommands(runtime);
 };

package/src/commands/renderers.ts

@@ -15,6 +15,8 @@ import {
 } from "../tools/render-common.ts";
 import { formatAdvisorUsage } from "../usage.ts";
 import { ManualAdvisorProgressComponent } from "./manual-progress.ts";
+import { formatAdvisorStats } from "./outcome-stats.ts";
+import type { AdvisorStatsDetails } from "./outcome-stats.ts";
 import type { CommandRuntime } from "./types.ts";
 
 interface ManualCallEntryDetails {
@@ -24,10 +26,22 @@ interface ManualCallEntryDetails {
 
 interface ManualResultDetails {
   advisor?: string;
+  agentRulesBytes?: number;
+  preferenceBytes?: number;
   text?: string;
   usage?: unknown;
 }
 
+const formatByteSize = (bytes: number) => {
+  if (bytes >= 1024 * 1024) {
+    return `${(bytes / (1024 * 1024)).toFixed(1)} MiB`;
+  }
+  if (bytes >= 1024) {
+    return `${Math.round(bytes / 1024)} KiB`;
+  }
+  return `${bytes} B`;
+};
+
 const manualCallRenderer =
   (runtime: CommandRuntime): EntryRenderer<ManualCallEntryDetails> =>
   (entry, { expanded }, theme) => {
@@ -70,12 +84,45 @@ const manualResultRenderer: MessageRenderer<ManualResultDetails> = (
       box.addChild(new Text(theme.fg("dim", `  Usage: ${usage}`), 0, 0));
     }
   }
+  const attachments = [
+    details?.agentRulesBytes
+      ? `AGENTS.md rules attached · ${formatByteSize(details.agentRulesBytes)}`
+      : undefined,
+    details?.preferenceBytes
+      ? `Project preferences attached · ${formatByteSize(details.preferenceBytes)}`
+      : undefined,
+  ].filter((value): value is string => value !== undefined);
+  if (attachments.length) {
+    box.addChild(
+      new Text(theme.fg("dim", `  ${attachments.join(" · ")}`), 0, 0)
+    );
+  }
   box.addChild(
     new Markdown(adviceForDisplay(advice, expanded), 0, 0, getMarkdownTheme())
   );
   return box;
 };
 
+const outcomeStatsRenderer: MessageRenderer<AdvisorStatsDetails> = (
+  message,
+  _options,
+  theme
+) => {
+  const box = new Box(1, 1, (text) => theme.bg("customMessageBg", text));
+  box.addChild(
+    new Text(theme.fg("accent", theme.bold("◆ ADVISOR · OUTCOME STATS")), 0, 0)
+  );
+  box.addChild(
+    new Markdown(
+      formatAdvisorStats(message.details ?? { malformedLines: 0 }),
+      0,
+      0,
+      getMarkdownTheme()
+    )
+  );
+  return box;
+};
+
 export const registerCommandRenderers = (runtime: CommandRuntime) => {
   runtime.pi.registerEntryRenderer?.(
     "advisor-manual-call",
@@ -85,4 +132,8 @@ export const registerCommandRenderers = (runtime: CommandRuntime) => {
     "advisor-manual-result",
     manualResultRenderer
   );
+  runtime.pi.registerMessageRenderer?.(
+    "advisor-stats-result",
+    outcomeStatsRenderer
+  );
 };

package/src/commands/settings-persistence.ts

@@ -3,6 +3,7 @@ import type { ExtensionContext } from "@earendil-works/pi-coding-agent";
 import {
   getAdvisorSettings,
   getPersistedModelRefs,
+  setAdvisorAgentsMdContextRef,
   setAdvisorAutoLoopGateRef,
   setAdvisorBlockOnBlockedRef,
   setAdvisorCollapseResponsesRef,
@@ -126,6 +127,7 @@ const applyDisclosureSettings = (settings: AdvisorSettings) => {
   setAdvisorGitContextRef(settings.gitContext ?? "summary");
   setAdvisorGitContextMaxCharsRef(settings.gitContextMaxChars ?? 20_000);
   setAdvisorToolPoliciesRef(settings.toolPolicies ?? {});
+  setAdvisorAgentsMdContextRef(settings.agentsMdContext ?? true);
   setAdvisorTrackedFileContentRef(settings.trackedFileContent ?? false);
   setAdvisorUntrackedContentRef(settings.untrackedContent ?? false);
   setAdvisorOutcomeLoggingRef(settings.outcomeLogging ?? false);

package/src/commands/types.ts

@@ -19,10 +19,11 @@ export type ManualConsult = (
   gitContext?: GitContextLevel
 ) => Promise<{
   adviceId?: string;
-  markdown: string;
-  thinkingText: string;
+  agentRulesBytes?: number;
   draftBytes?: number;
+  markdown: string;
   preferenceBytes?: number;
+  thinkingText: string;
   usage?: unknown;
 }>;
 

package/src/config/defaults.ts

@@ -1,4 +1,5 @@
 import {
+  setAdvisorAgentsMdContextRef,
   setAdvisorAutoLoopGateRef,
   setAdvisorBlockOnBlockedRef,
   setAdvisorCollapseResponsesRef,
@@ -68,6 +69,7 @@ import {
 export const resetDefaults = () => {
   setExecutorRef("");
   setAdvisorRef("");
+  setAdvisorAgentsMdContextRef(true);
   setPersistedModelRefs(undefined, undefined);
   setExecutorEffortRef(undefined);
   setAdvisorEffortRef(undefined);
@@ -139,6 +141,11 @@ export const applyConfig = (config: AdvisorConfig) => {
   applyNonEmptyStringConfig(config.advisor, setAdvisorRef);
   applyNonEmptyStringConfig(config.executorEffort, setExecutorEffortRef);
   applyNonEmptyStringConfig(config.advisorEffort, setAdvisorEffortRef);
+  applyOptionalConfig(
+    config,
+    "advisorAgentsMdContext",
+    setAdvisorAgentsMdContextRef
+  );
   applyOptionalConfig(config, "contextMaxChars", setContextMaxCharsRef);
   applyOptionalConfig(config, "advisorPlanGate", setAdvisorPlanGateRef);
   applyOptionalConfig(config, "advisorFailureGate", setAdvisorFailureGateRef);

package/src/config/schema.ts

@@ -2,6 +2,7 @@ import { isString } from "../content-utils.ts";
 import type { JsonValue } from "../content-utils.ts";
 import { GIT_CONTEXT_LEVELS, isValidGitContextLevel } from "../git.ts";
 import {
+  advisorAgentsMdContextRef,
   advisorAutoLoopGateRef,
   advisorBlockOnBlockedRef,
   advisorCollapseResponsesRef,
@@ -171,6 +172,12 @@ export const CONFIG_SCHEMA = {
     persisted: true,
     type: "string",
   },
+  advisorAgentsMdContext: {
+    accepted: "true or false",
+    current: () => advisorAgentsMdContextRef,
+    persisted: true,
+    type: "boolean",
+  },
   advisorAutoLoopGate: {
     accepted: "true or false",
     current: () => advisorAutoLoopGateRef,

package/src/config/state.ts

@@ -25,6 +25,7 @@ import type {
 // An empty ref means no model has been selected yet.
 export let executorRef = "";
 export let advisorRef = "";
+export let advisorAgentsMdContextRef = true;
 let persistedExecutorRef: string | undefined;
 let persistedAdvisorRef: string | undefined;
 export let executorEffortRef: string | undefined;
@@ -80,6 +81,9 @@ export const setExecutorRef = (ref: string) => {
 export const setAdvisorRef = (ref: string) => {
   advisorRef = ref;
 };
+export const setAdvisorAgentsMdContextRef = (enabled: boolean) => {
+  advisorAgentsMdContextRef = enabled;
+};
 /** Returns model refs explicitly persisted in the global Advisor config. */
 export const getPersistedModelRefs = () => ({
   advisor: persistedAdvisorRef,
@@ -234,6 +238,7 @@ export const setShowUsageFooterRef = (enabled: boolean) => {
  * imported mutable bindings, which can be snapshotted by extension loaders.
  */
 export const getAdvisorSettings = () => ({
+  agentsMdContext: advisorAgentsMdContextRef,
   alwaysOn: alwaysOnRef,
   autoLoopGate: advisorAutoLoopGateRef,
   blockOnBlocked: advisorBlockOnBlockedRef,

package/src/config/types.ts

@@ -60,6 +60,7 @@ export const GATE_FAILURE_MODES: GateFailureMode[] = [
 
 export interface AdvisorConfig {
   advisor?: string;
+  advisorAgentsMdContext?: boolean;
   advisorAutoLoopGate?: boolean;
   advisorBlockOnBlocked?: boolean;
   advisorCollapseResponses?: boolean;

package/src/config.ts

@@ -1,6 +1,7 @@
 // biome-ignore lint/performance/noBarrelFile: this facade intentionally preserves the public config module contract.
 export { parseArgs } from "./config/args.ts";
 export {
+  advisorAgentsMdContextRef,
   advisorAutoLoopGateRef,
   advisorBlockOnBlockedRef,
   advisorCollapseResponsesRef,
@@ -34,6 +35,7 @@ export {
   getAdvisorSettings,
   getPersistedModelRefs,
   isSimpleMode,
+  setAdvisorAgentsMdContextRef,
   setAdvisorAutoLoopGateRef,
   setAdvisorBlockOnBlockedRef,
   setAdvisorCollapseResponsesRef,

package/src/outcome-stats.ts

@@ -0,0 +1,154 @@
+import { readFile } from "node:fs/promises";
+
+import { isRecord, isString } from "./content-utils.ts";
+import {
+  ADOPTIONS,
+  OUTCOME_TRIGGERS,
+  VALIDATIONS,
+  outcomeLogPath,
+} from "./outcomes.ts";
+import type {
+  OutcomeAdoption,
+  OutcomeRecord,
+  OutcomeTrigger,
+} from "./outcomes.ts";
+
+interface OutcomeValidationStats {
+  failed: number;
+  notRun: number;
+  passed: number;
+  passRate?: number;
+  total: number;
+  unknown: number;
+}
+export interface OutcomeStats {
+  adoption: Record<OutcomeAdoption, number>;
+  byTrigger: Record<OutcomeTrigger, number>;
+  distinctAdvices: number;
+  firstTimestamp?: string;
+  lastTimestamp?: string;
+  malformedLines: number;
+  total: number;
+  validationByAdoption: Record<OutcomeAdoption, OutcomeValidationStats>;
+}
+export interface OutcomeLogRead {
+  malformedLines: number;
+  missing: boolean;
+  records: OutcomeRecord[];
+}
+
+const ADOPTION_SET = new Set<string>(ADOPTIONS);
+const VALIDATION_SET = new Set<string>(VALIDATIONS);
+const TRIGGER_SET = new Set<string>(OUTCOME_TRIGGERS);
+const isErrnoException = (error: unknown): error is NodeJS.ErrnoException =>
+  error instanceof Error && "code" in error;
+
+const isOutcomeRecord = (value: unknown): value is OutcomeRecord =>
+  isRecord(value) &&
+  value.v === 1 &&
+  isString(value.adoption) &&
+  ADOPTION_SET.has(value.adoption) &&
+  isString(value.adviceHash) &&
+  isString(value.timestamp) &&
+  Number.isFinite(Date.parse(value.timestamp)) &&
+  isString(value.trigger) &&
+  TRIGGER_SET.has(value.trigger) &&
+  isString(value.validationStatus) &&
+  VALIDATION_SET.has(value.validationStatus);
+
+export const parseOutcomeLog = (text: string): OutcomeLogRead => {
+  const records: OutcomeRecord[] = [];
+  let malformedLines = 0;
+  for (const line of text.split(/\r?\n/u)) {
+    if (!line.trim()) {
+      continue;
+    }
+    try {
+      const value: unknown = JSON.parse(line);
+      if (isOutcomeRecord(value)) {
+        records.push(value);
+      } else {
+        malformedLines += 1;
+      }
+    } catch {
+      malformedLines += 1;
+    }
+  }
+  return { malformedLines, missing: false, records };
+};
+
+export const readOutcomeLog = async (
+  path = outcomeLogPath()
+): Promise<OutcomeLogRead> => {
+  try {
+    return parseOutcomeLog(await readFile(path, "utf-8"));
+  } catch (error) {
+    if (isErrnoException(error) && error.code === "ENOENT") {
+      return { malformedLines: 0, missing: true, records: [] };
+    }
+    throw error;
+  }
+};
+
+const emptyValidationStats = (): OutcomeValidationStats => ({
+  failed: 0,
+  notRun: 0,
+  passed: 0,
+  total: 0,
+  unknown: 0,
+});
+
+export const aggregateOutcomeRecords = (
+  records: readonly OutcomeRecord[],
+  malformedLines = 0
+): OutcomeStats => {
+  // SAFETY: ADOPTIONS enumerates every key in the returned adoption record.
+  const adoption = Object.fromEntries(
+    ADOPTIONS.map((value) => [value, 0])
+  ) as Record<OutcomeAdoption, number>;
+  // SAFETY: OUTCOME_TRIGGERS enumerates every key in the returned trigger record.
+  const byTrigger = Object.fromEntries(
+    OUTCOME_TRIGGERS.map((value) => [value, 0])
+  ) as Record<OutcomeTrigger, number>;
+  // SAFETY: ADOPTIONS enumerates every key and each value gets a fresh stats object.
+  const validationByAdoption = Object.fromEntries(
+    ADOPTIONS.map((value) => [value, emptyValidationStats()])
+  ) as Record<OutcomeAdoption, OutcomeValidationStats>;
+  const adviceHashes = new Set<string>();
+  let first: { timestamp: string; time: number } | undefined;
+  let last: { timestamp: string; time: number } | undefined;
+
+  for (const record of records) {
+    adoption[record.adoption] += 1;
+    byTrigger[record.trigger] += 1;
+    adviceHashes.add(record.adviceHash);
+    const validation = validationByAdoption[record.adoption];
+    validation.total += 1;
+    validation[
+      record.validationStatus === "not-run" ? "notRun" : record.validationStatus
+    ] += 1;
+    const time = Date.parse(record.timestamp);
+    if (!first || time < first.time) {
+      first = { time, timestamp: record.timestamp };
+    }
+    if (!last || time > last.time) {
+      last = { time, timestamp: record.timestamp };
+    }
+  }
+  for (const value of Object.values(validationByAdoption)) {
+    const validated = value.passed + value.failed;
+    if (validated > 0) {
+      value.passRate = value.passed / validated;
+    }
+  }
+  return {
+    adoption,
+    byTrigger,
+    distinctAdvices: adviceHashes.size,
+    firstTimestamp: first?.timestamp,
+    lastTimestamp: last?.timestamp,
+    malformedLines,
+    total: records.length,
+    validationByAdoption,
+  };
+};

package/src/outcomes.ts

@@ -17,7 +17,7 @@ import { getAgentDir } from "@earendil-works/pi-coding-agent";
 
 export type OutcomeAdoption = "followed" | "not-followed" | "unknown";
 export type OutcomeValidation = "passed" | "failed" | "not-run" | "unknown";
-type OutcomeTrigger =
+export type OutcomeTrigger =
   | "manual"
   | "executor-requested"
   | "turn-gate"
@@ -33,6 +33,12 @@ export const VALIDATIONS: OutcomeValidation[] = [
   "not-run",
   "unknown",
 ];
+export const OUTCOME_TRIGGERS: OutcomeTrigger[] = [
+  "manual",
+  "executor-requested",
+  "turn-gate",
+  "repeated-tool-call",
+];
 export interface OutcomeRecord {
   adoption: OutcomeAdoption;
   adviceHash: string;
@@ -42,6 +48,7 @@ export interface OutcomeRecord {
   validationStatus: OutcomeValidation;
 }
 const MAX_LOG_BYTES = 1024 * 1024;
+export const OUTCOME_LOG_MAX_BYTES = MAX_LOG_BYTES;
 const statePath = () => join(getAgentDir(), "advisor-outcomes-salt");
 export const outcomeLogPath = () =>
   join(getAgentDir(), "advisor-outcomes.jsonl");