pi-fluency 0.1.0

package/extensions/pi-fluency/collector.ts

@@ -0,0 +1,102 @@
+import { createHash } from "node:crypto";
+import { sanitizeCollectedInput } from "./sanitize.js";
+import type { CollectedPrompt } from "./types.js";
+
+const SECRET_PATTERNS = [
+  /-----BEGIN(?: [A-Z0-9]+)* PRIVATE KEY-----[\s\S]*?-----END(?: [A-Z0-9]+)* PRIVATE KEY-----/gi,
+  /\bAuthorization\s*:\s*(?:Bearer|Basic)\s+[^\s,;]+/gi,
+  /\bBearer\s+[^\s,;]+/gi,
+  /\bgithub_pat_[A-Za-z0-9_]{8,}\b/g,
+  /\bgh[opusr]_[A-Za-z0-9_]{8,}\b/g,
+  /\b(?:AKIA|ASIA)[A-Z0-9]{16}\b/g,
+  /(?<![A-Za-z0-9_-])eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+(?![A-Za-z0-9_-])/g,
+  /\bsk-[A-Za-z0-9_-]{20,}\b/g,
+  /\b[a-z][a-z0-9+.-]*:\/\/[^\s/@:]+:[^@\s/]+@/gi,
+  /\b(?:api\s*[_-]?\s*key|token|secret|password|client[_-]?secret|database[_-]?url|[A-Z][A-Z0-9_]*(?:API[_-]?KEY|ACCESS[_-]?KEY|SECRET[_-]?KEY|TOKEN|SECRET|PASSWORD))\s*(?::|=|\s)\s*[^\s,;]+/gi,
+];
+
+function stripBlockCode(text: string): string {
+  const proseLines: string[] = [];
+  let fence: { marker: "`" | "~"; length: number } | undefined;
+
+  for (const line of text.split("\n")) {
+    const withoutCarriageReturn = line.replace(/\r$/, "");
+
+    if (fence) {
+      const closing = /^ {0,3}(`+|~+)[ \t]*$/.exec(withoutCarriageReturn);
+      if (closing?.[1]?.startsWith(fence.marker) && closing[1].length >= fence.length) {
+        fence = undefined;
+      }
+      continue;
+    }
+
+    const opening = /^ {0,3}(`{3,}|~{3,})/.exec(withoutCarriageReturn);
+    if (opening?.[1]) {
+      fence = {
+        marker: opening[1][0] as "`" | "~",
+        length: opening[1].length,
+      };
+      continue;
+    }
+
+    if (/^(?: {4,}|\t)/.test(withoutCarriageReturn)) continue;
+    proseLines.push(line);
+  }
+
+  return proseLines.join("\n");
+}
+
+function stripInlineCode(text: string): string {
+  let prose = "";
+  let index = 0;
+
+  while (index < text.length) {
+    if (text[index] !== "`") {
+      prose += text[index];
+      index += 1;
+      continue;
+    }
+
+    let openerEnd = index;
+    while (text[openerEnd] === "`") openerEnd += 1;
+    const delimiterLength = openerEnd - index;
+    let searchIndex = openerEnd;
+    let closingEnd: number | undefined;
+
+    while (searchIndex < text.length) {
+      if (text[searchIndex] !== "`") {
+        searchIndex += 1;
+        continue;
+      }
+
+      let candidateEnd = searchIndex;
+      while (text[candidateEnd] === "`") candidateEnd += 1;
+      if (candidateEnd - searchIndex === delimiterLength) {
+        closingEnd = candidateEnd;
+        break;
+      }
+      searchIndex = candidateEnd;
+    }
+
+    prose += " ";
+    if (closingEnd === undefined) break;
+    index = closingEnd;
+  }
+
+  return prose;
+}
+
+export function collectPrompt(text: string, observedAt = Date.now()): CollectedPrompt | undefined {
+  if (text.trimStart().startsWith("/")) return undefined;
+
+  let prose = stripInlineCode(stripBlockCode(sanitizeCollectedInput(text)));
+  for (const pattern of SECRET_PATTERNS) prose = prose.replace(pattern, "[REDACTED]");
+  prose = prose.replace(/\s+/g, " ").trim();
+  if (prose.length < 8 || (prose.match(/\p{L}/gu)?.length ?? 0) < 3) return undefined;
+
+  return {
+    prose,
+    observedAt,
+    promptHash: createHash("sha256").update(prose).digest("hex"),
+  };
+}

package/extensions/pi-fluency/context.ts

@@ -0,0 +1,65 @@
+import type { AnalyzerMistake, RawAnalyzerMistake } from "./types.js";
+
+const MAX_EXCERPT_LENGTH = 500;
+
+export function materializeMistake(prose: string, raw: RawAnalyzerMistake): AnalyzerMistake {
+  if (raw.original.length === 0) throw new Error("Source quote not found exactly once");
+
+  const expansion = raw.correction.length - raw.original.length;
+  const sourceLimit = Math.min(MAX_EXCERPT_LENGTH, MAX_EXCERPT_LENGTH - expansion);
+  if (
+    raw.original.length > MAX_EXCERPT_LENGTH
+    || raw.correction.length > MAX_EXCERPT_LENGTH
+    || sourceLimit < raw.original.length
+  ) {
+    throw new Error("Invalid analysis result");
+  }
+
+  const first = prose.indexOf(raw.original);
+  const second = first < 0 ? -1 : prose.indexOf(raw.original, first + 1);
+  if (first < 0 || second >= 0) throw new Error("Source quote not found exactly once");
+
+  const segments = [...new Intl.Segmenter("en", { granularity: "sentence" }).segment(prose)];
+  const sentenceIndex = segments.findIndex(
+    (segment) => first >= segment.index && first < segment.index + segment.segment.length,
+  );
+  if (sentenceIndex < 0) throw new Error("Could not derive sentence context");
+
+  const startIndex = raw.contextScope === "previous-and-current"
+    ? Math.max(0, sentenceIndex - 1)
+    : sentenceIndex;
+  const endIndex = raw.contextScope === "current-and-next"
+    ? Math.min(segments.length - 1, sentenceIndex + 1)
+    : sentenceIndex;
+  const excerptStart = segments[startIndex]!.index;
+  const endSegment = segments[endIndex]!;
+  const excerptEnd = endSegment.index + endSegment.segment.length;
+  let sourceExcerpt = prose.slice(excerptStart, excerptEnd).trim();
+
+  if (sourceExcerpt.length > sourceLimit) {
+    const localIndex = sourceExcerpt.indexOf(raw.original);
+    if (localIndex < 0) throw new Error("Invalid analysis result");
+
+    const earliestWindowStart = Math.max(0, localIndex + raw.original.length - sourceLimit);
+    const latestWindowStart = Math.min(localIndex, sourceExcerpt.length - sourceLimit);
+    const preferredWindowStart = localIndex - 200;
+    const windowStart = Math.min(
+      latestWindowStart,
+      Math.max(earliestWindowStart, preferredWindowStart),
+    );
+    sourceExcerpt = sourceExcerpt.slice(windowStart, windowStart + sourceLimit);
+  }
+
+  const localIndex = sourceExcerpt.indexOf(raw.original);
+  if (localIndex < 0) throw new Error("Invalid analysis result");
+  const correctedExcerpt = sourceExcerpt.slice(0, localIndex)
+    + raw.correction
+    + sourceExcerpt.slice(localIndex + raw.original.length);
+  if (
+    sourceExcerpt.length > MAX_EXCERPT_LENGTH
+    || correctedExcerpt.length > MAX_EXCERPT_LENGTH
+  ) {
+    throw new Error("Invalid analysis result");
+  }
+  return { ...raw, sourceExcerpt, correctedExcerpt };
+}

package/extensions/pi-fluency/diff.ts

@@ -0,0 +1,147 @@
+import { visibleWidth } from "@earendil-works/pi-tui";
+
+export interface CompactDiffInput {
+  sourceExcerpt: string;
+  correctedExcerpt: string;
+  original: string;
+  correction: string;
+}
+
+export interface CompactDiffStyles {
+  deletion(text: string): string;
+  insertion(text: string): string;
+}
+
+interface Token {
+  text: string;
+  start: number;
+  end: number;
+}
+
+const TOKEN = /\s+|[\p{L}\p{N}'’-]+|[^\s]/gu;
+export interface CompactDiffFallback {
+  source: string;
+  correction: string;
+}
+
+const fallbackOutputs = new WeakMap<string[], CompactDiffFallback>();
+
+function tokenize(text: string): Token[] {
+  return [...text.matchAll(TOKEN)].map((match) => ({
+    text: match[0],
+    start: match.index,
+    end: match.index + match[0].length,
+  }));
+}
+
+function verifiedOffset(input: CompactDiffInput): number | undefined {
+  if (input.original.length === 0) return undefined;
+  let offset = input.sourceExcerpt.indexOf(input.original);
+  while (offset >= 0) {
+    const candidate = input.sourceExcerpt.slice(0, offset)
+      + input.correction
+      + input.sourceExcerpt.slice(offset + input.original.length);
+    if (candidate === input.correctedExcerpt) return offset;
+    offset = input.sourceExcerpt.indexOf(input.original, offset + 1);
+  }
+  return undefined;
+}
+
+function visibleWhitespace(text: string): string {
+  return [...text].map((character) => {
+    if (character === "\t") return "⇥";
+    if (character === "\n" || character === "\r") return "↵";
+    return "␠";
+  }).join("");
+}
+
+function differingTokenSpan(source: string, corrected: string): CompactDiffFallback {
+  const sourceTokens = tokenize(source);
+  const correctedTokens = tokenize(corrected);
+  let prefix = 0;
+  while (
+    prefix < sourceTokens.length
+    && prefix < correctedTokens.length
+    && sourceTokens[prefix]!.text === correctedTokens[prefix]!.text
+  ) prefix += 1;
+
+  let suffix = 0;
+  while (
+    suffix < sourceTokens.length - prefix
+    && suffix < correctedTokens.length - prefix
+    && sourceTokens[sourceTokens.length - 1 - suffix]!.text === correctedTokens[correctedTokens.length - 1 - suffix]!.text
+  ) suffix += 1;
+
+  const span = (text: string, tokens: Token[]): string => {
+    const first = tokens[prefix];
+    const last = tokens[tokens.length - suffix - 1];
+    if (!first || !last || last.end < first.start) return "∅";
+    const raw = text.slice(first.start, last.end);
+    if (raw.length === 0) return "∅";
+    if (/^\s+$/u.test(raw)) return visibleWhitespace(raw);
+    return raw.trim();
+  };
+  return { source: span(source, sourceTokens), correction: span(corrected, correctedTokens) };
+}
+
+function fallback(input: CompactDiffInput): string[] {
+  const fallback = differingTokenSpan(input.sourceExcerpt, input.correctedExcerpt);
+  const lines = [fallback.source, `└─ ${fallback.correction}`];
+  fallbackOutputs.set(lines, fallback);
+  return lines;
+}
+
+/** Return layout metadata only for the unverifiable/legacy token-diff fallback. */
+export function compactDiffFallback(lines: string[]): CompactDiffFallback | undefined {
+  return fallbackOutputs.get(lines);
+}
+
+/** Render one deterministic, compact edit without mutating either excerpt. */
+export function renderCompactDiff(input: CompactDiffInput, styles: CompactDiffStyles): string[] {
+  const excerptOffset = verifiedOffset(input);
+  if (excerptOffset === undefined) return fallback(input);
+
+  const removed = tokenize(input.original);
+  const added = tokenize(input.correction);
+  let prefix = 0;
+  while (prefix < removed.length && prefix < added.length && removed[prefix]!.text === added[prefix]!.text) {
+    prefix += 1;
+  }
+  let suffix = 0;
+  while (
+    suffix < removed.length - prefix
+    && suffix < added.length - prefix
+    && removed[removed.length - 1 - suffix]!.text === added[added.length - 1 - suffix]!.text
+  ) suffix += 1;
+
+  const removedStart = prefix < removed.length ? removed[prefix]!.start : input.original.length;
+  const removedEndIndex = removed.length - suffix - 1;
+  const removedEnd = removedEndIndex >= prefix ? removed[removedEndIndex]!.end : removedStart;
+  const addedStart = prefix < added.length ? added[prefix]!.start : input.correction.length;
+  const addedEndIndex = added.length - suffix - 1;
+  const addedEnd = addedEndIndex >= prefix ? added[addedEndIndex]!.end : addedStart;
+  const removedText = input.original.slice(removedStart, removedEnd);
+  const addedText = input.correction.slice(addedStart, addedEnd);
+
+  if (removedText && addedText) {
+    const annotationOffset = excerptOffset + removedStart;
+    return [
+      input.sourceExcerpt,
+      `${" ".repeat(visibleWidth(input.sourceExcerpt.slice(0, annotationOffset)))}└─ ${addedText}`,
+    ];
+  }
+
+  if (removedText) {
+    const start = excerptOffset + removedStart;
+    const end = excerptOffset + removedEnd;
+    return [input.sourceExcerpt.slice(0, start) + styles.deletion(removedText) + input.sourceExcerpt.slice(end)];
+  }
+
+  if (addedText) {
+    const start = excerptOffset + addedStart;
+    const end = excerptOffset + addedEnd;
+    return [input.correctedExcerpt.slice(0, start) + styles.insertion(addedText) + input.correctedExcerpt.slice(end)];
+  }
+
+  return input.sourceExcerpt === input.correctedExcerpt ? [input.sourceExcerpt] : fallback(input);
+}

package/extensions/pi-fluency/generation-marker.ts

@@ -0,0 +1,51 @@
+const HISTORY_GENERATION = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
+
+export interface HistoryGenerationMarker {
+  generation: string;
+  resetPending: boolean;
+  legacy: boolean;
+}
+
+function invalid(): never {
+  throw new Error("Invalid history generation");
+}
+
+function validGeneration(value: unknown): value is string {
+  return typeof value === "string" && HISTORY_GENERATION.test(value);
+}
+
+export function decodeHistoryGenerationMarker(serialized: string): HistoryGenerationMarker {
+  const trimmed = serialized.trim();
+  if (validGeneration(trimmed)) {
+    return { generation: trimmed, resetPending: false, legacy: true };
+  }
+
+  let value: unknown;
+  try {
+    value = JSON.parse(trimmed) as unknown;
+  } catch {
+    return invalid();
+  }
+  if (!value || typeof value !== "object" || Array.isArray(value)) return invalid();
+  const marker = value as Record<string, unknown>;
+  const keys = Object.keys(marker).sort();
+  if (
+    keys.length !== 2
+    || keys[0] !== "generation"
+    || keys[1] !== "resetPending"
+    || !validGeneration(marker.generation)
+    || typeof marker.resetPending !== "boolean"
+  ) return invalid();
+  return { generation: marker.generation, resetPending: marker.resetPending, legacy: false };
+}
+
+export function encodeHistoryGenerationMarker(marker: {
+  generation: string;
+  resetPending: boolean;
+}): string {
+  if (!validGeneration(marker.generation)) return invalid();
+  return `${JSON.stringify({
+    generation: marker.generation,
+    resetPending: marker.resetPending,
+  })}\n`;
+}

package/extensions/pi-fluency/history-codec.ts

@@ -0,0 +1,266 @@
+import { isErrantErrorType } from "./taxonomy.js";
+import {
+  HISTORY_SCHEMA_VERSION,
+  type AnalysisResult,
+  type AnalyzerMistake,
+  type ContextScope,
+  type DemonstratedFix,
+  type EnglishObservation,
+  type FluencyEvent,
+  type MistakeOccurrence,
+  type SnapshotPattern,
+} from "./types.js";
+
+const EVENT_TYPES = new Set(["analysis", "review", "snapshot"]);
+const CONTEXT_SCOPES = new Set<ContextScope>(["sentence", "previous-and-current", "current-and-next"]);
+const PATTERN_KEY = /^[a-z]+(?:[.-][a-z0-9]+)+$/;
+
+export class HistorySchemaMismatchError extends Error {
+  constructor() {
+    super("Unsupported fluency history schema");
+    this.name = "HistorySchemaMismatchError";
+  }
+}
+
+function invalid(): never {
+  throw new Error("Invalid schema-v4 history event");
+}
+
+function record(value: unknown): Record<string, unknown> {
+  if (!value || typeof value !== "object" || Array.isArray(value)) return invalid();
+  return value as Record<string, unknown>;
+}
+
+function text(value: unknown, allowEmpty = false): string {
+  if (typeof value !== "string" || (!allowEmpty && value.length === 0)) return invalid();
+  return value;
+}
+
+function finite(value: unknown): number {
+  if (typeof value !== "number" || !Number.isFinite(value)) return invalid();
+  return value;
+}
+
+function timestamp(value: unknown): number {
+  const decoded = finite(value);
+  if (!Number.isFinite(new Date(decoded).getTime())) return invalid();
+  return decoded;
+}
+
+function nonnegativeInteger(value: unknown): number {
+  if (typeof value !== "number" || !Number.isInteger(value) || value < 0) return invalid();
+  return value;
+}
+
+function confidence(value: unknown): number {
+  const decoded = finite(value);
+  if (decoded < 0 || decoded > 1) return invalid();
+  return decoded;
+}
+
+function stringArray(value: unknown): string[] {
+  if (!Array.isArray(value)) return invalid();
+  const decoded = value.map((item) => text(item));
+  if (new Set(decoded).size !== decoded.length) return invalid();
+  return decoded;
+}
+
+export function isLocalDate(value: unknown): value is string {
+  if (typeof value !== "string" || !/^\d{4}-\d{2}-\d{2}$/.test(value)) return false;
+  const timestamp = Date.parse(`${value}T00:00:00.000Z`);
+  return Number.isFinite(timestamp) && new Date(timestamp).toISOString().slice(0, 10) === value;
+}
+
+function localDate(value: unknown): string {
+  if (!isLocalDate(value)) return invalid();
+  return value;
+}
+
+function decodeMistake(value: unknown): AnalyzerMistake {
+  const candidate = record(value);
+  const contextScope = text(candidate.contextScope);
+  if (!CONTEXT_SCOPES.has(contextScope as ContextScope)) return invalid();
+  const errorType = text(candidate.errorType);
+  if (!isErrantErrorType(errorType)) return invalid();
+  const patternKey = text(candidate.patternKey);
+  if (!PATTERN_KEY.test(patternKey)) return invalid();
+  return {
+    original: text(candidate.original),
+    correction: text(candidate.correction, true),
+    contextScope: contextScope as ContextScope,
+    sourceExcerpt: text(candidate.sourceExcerpt),
+    correctedExcerpt: text(candidate.correctedExcerpt, true),
+    explanation: text(candidate.explanation),
+    errorType,
+    patternKey,
+    confidence: confidence(candidate.confidence),
+  };
+}
+
+function decodeFix(value: unknown): DemonstratedFix {
+  const candidate = record(value);
+  const patternKey = text(candidate.patternKey);
+  if (!PATTERN_KEY.test(patternKey)) return invalid();
+  return {
+    patternKey,
+    evidence: candidate.evidence === undefined ? "" : text(candidate.evidence),
+    confidence: confidence(candidate.confidence),
+  };
+}
+
+function decodeAnalysisResult(value: unknown): AnalysisResult {
+  const candidate = record(value);
+  if (candidate.schemaVersion !== 3 || (candidate.language !== "en" && candidate.language !== "other")) return invalid();
+  if (!Array.isArray(candidate.mistakes) || !Array.isArray(candidate.demonstratedFixes)) return invalid();
+  const mistakes = candidate.mistakes.map(decodeMistake);
+  const demonstratedFixes = candidate.demonstratedFixes.map(decodeFix);
+  if (candidate.language === "other" && (mistakes.length > 0 || demonstratedFixes.length > 0)) return invalid();
+  return { schemaVersion: 3, language: candidate.language, mistakes, demonstratedFixes };
+}
+
+function decodePattern(value: unknown): SnapshotPattern {
+  const candidate = record(value);
+  const errorType = text(candidate.errorType);
+  if (!isErrantErrorType(errorType)) return invalid();
+  const patternKey = text(candidate.patternKey);
+  if (!PATTERN_KEY.test(patternKey)) return invalid();
+  return {
+    id: text(candidate.id),
+    patternKey,
+    original: text(candidate.original),
+    correction: text(candidate.correction, true),
+    sourceExcerpt: text(candidate.sourceExcerpt),
+    correctedExcerpt: text(candidate.correctedExcerpt, true),
+    explanation: text(candidate.explanation),
+    errorType,
+    confidence: confidence(candidate.confidence),
+    firstSeenAt: timestamp(candidate.firstSeenAt),
+    lastSeenAt: timestamp(candidate.lastSeenAt),
+    occurrenceCount: nonnegativeInteger(candidate.occurrenceCount),
+    demonstratedFixCount: nonnegativeInteger(candidate.demonstratedFixCount),
+  };
+}
+
+function decodeObservation(value: unknown): EnglishObservation {
+  const candidate = record(value);
+  return {
+    promptHash: text(candidate.promptHash),
+    observedAt: timestamp(candidate.observedAt),
+    localDate: localDate(candidate.localDate),
+    wordCount: nonnegativeInteger(candidate.wordCount),
+    occurrenceIds: stringArray(candidate.occurrenceIds),
+  };
+}
+
+function decodeOccurrence(value: unknown): MistakeOccurrence {
+  const candidate = record(value);
+  if (candidate.decision !== "pending" && candidate.decision !== "accepted" && candidate.decision !== "dismissed") return invalid();
+  return {
+    id: text(candidate.id),
+    promptHash: text(candidate.promptHash),
+    patternId: text(candidate.patternId),
+    patternKey: text(candidate.patternKey),
+    observedAt: timestamp(candidate.observedAt),
+    localDate: localDate(candidate.localDate),
+    decision: candidate.decision,
+  };
+}
+
+function decodeAnalysis(candidate: Record<string, unknown>): FluencyEvent {
+  const prompt = record(candidate.prompt);
+  return {
+    schemaVersion: HISTORY_SCHEMA_VERSION,
+    type: "analysis",
+    at: timestamp(candidate.at),
+    prompt: {
+      promptHash: text(prompt.promptHash),
+      observedAt: timestamp(prompt.observedAt),
+      prose: prompt.prose === undefined ? "" : text(prompt.prose, true),
+    },
+    wordCount: nonnegativeInteger(candidate.wordCount),
+    result: decodeAnalysisResult(candidate.result),
+  };
+}
+
+function decodeReview(candidate: Record<string, unknown>): FluencyEvent {
+  if (candidate.decision !== "accepted" && candidate.decision !== "dismissed") return invalid();
+  const occurrenceIds = stringArray(candidate.occurrenceIds);
+  if (occurrenceIds.length === 0) return invalid();
+  return {
+    schemaVersion: HISTORY_SCHEMA_VERSION,
+    type: "review",
+    at: timestamp(candidate.at),
+    occurrenceIds,
+    decision: candidate.decision,
+  };
+}
+
+function decodeSnapshot(candidate: Record<string, unknown>): FluencyEvent {
+  if (
+    !Array.isArray(candidate.patterns)
+    || !Array.isArray(candidate.observations)
+    || !Array.isArray(candidate.occurrences)
+  ) return invalid();
+  const patterns = candidate.patterns.map(decodePattern);
+  const observations = candidate.observations.map(decodeObservation);
+  const occurrences = candidate.occurrences.map(decodeOccurrence);
+  const processedPromptHashes = stringArray(candidate.processedPromptHashes);
+
+  const patternById = new Map(patterns.map((pattern) => [pattern.id, pattern]));
+  const observationByHash = new Map(observations.map((observation) => [observation.promptHash, observation]));
+  const occurrenceById = new Map(occurrences.map((occurrence) => [occurrence.id, occurrence]));
+  if (
+    patternById.size !== patterns.length
+    || observationByHash.size !== observations.length
+    || occurrenceById.size !== occurrences.length
+  ) return invalid();
+
+  for (const observation of observations) {
+    for (const id of observation.occurrenceIds) {
+      const occurrence = occurrenceById.get(id);
+      if (!occurrence || occurrence.promptHash !== observation.promptHash) return invalid();
+    }
+  }
+  for (const occurrence of occurrences) {
+    const suffix = occurrence.id.slice(occurrence.promptHash.length + 1);
+    const observation = observationByHash.get(occurrence.promptHash);
+    const pattern = patternById.get(occurrence.patternId);
+    if (
+      !occurrence.id.startsWith(`${occurrence.promptHash}:`)
+      || !/^\d+$/.test(suffix)
+      || !observation?.occurrenceIds.includes(occurrence.id)
+      || occurrence.observedAt !== observation.observedAt
+      || occurrence.localDate !== observation.localDate
+      || pattern?.patternKey !== occurrence.patternKey
+    ) return invalid();
+  }
+  if (
+    observations.some((observation) => !processedPromptHashes.includes(observation.promptHash))
+  ) return invalid();
+
+  return {
+    schemaVersion: HISTORY_SCHEMA_VERSION,
+    type: "snapshot",
+    at: timestamp(candidate.at),
+    patterns,
+    observations,
+    occurrences,
+    processedPromptHashes,
+  };
+}
+
+export function decodeHistoryLine(value: unknown): FluencyEvent {
+  const candidate = record(value);
+  if (candidate.schemaVersion !== HISTORY_SCHEMA_VERSION) throw new HistorySchemaMismatchError();
+  if (typeof candidate.type !== "string" || !EVENT_TYPES.has(candidate.type)) return invalid();
+  if (candidate.type === "analysis") return decodeAnalysis(candidate);
+  if (candidate.type === "review") return decodeReview(candidate);
+  return decodeSnapshot(candidate);
+}
+
+export function encodeHistoryEvent(event: FluencyEvent): string {
+  return JSON.stringify(event, (key, value) =>
+    key === "prose" || key === "evidence"
+      ? undefined
+      : value);
+}