recallnest 1.4.1

package/src/scope-inventory.ts

@@ -0,0 +1,310 @@
+import { existsSync, readFileSync, readdirSync } from "node:fs";
+import { basename, join, resolve } from "node:path";
+
+import { suppressesLegacyScopeIssue } from "./legacy-scope-review.js";
+import { classifyLegacyScope, type LegacyScopeAudit, type LegacyScopeIssueKind, type MemoryStore } from "./store.js";
+
+export type ScopeInventoryLayer =
+  | "memories"
+  | "pins"
+  | "session-checkpoints"
+  | "workflow-observations";
+
+export interface ScopeInventorySample {
+  layer: ScopeInventoryLayer;
+  id: string;
+  kind: LegacyScopeIssueKind;
+  scope: string | null;
+  context: string;
+  preview: string;
+  recordedAt?: string;
+  sourcePath?: string;
+}
+
+export interface ScopeInventoryLayerReport {
+  layer: ScopeInventoryLayer;
+  scannedCount: number;
+  anomalyCount: number;
+  invalidCount: number;
+  reviewedCount: number;
+  counts: Record<LegacyScopeIssueKind, number>;
+  samples: ScopeInventorySample[];
+  recommendation: string;
+}
+
+export interface ScopeInventoryReport {
+  generatedAt: string;
+  sampleLimit: number;
+  totalScannedCount: number;
+  totalAnomalyCount: number;
+  totalInvalidCount: number;
+  totalReviewedCount: number;
+  layers: ScopeInventoryLayerReport[];
+}
+
+export interface ScopeInventoryOptions {
+  store: MemoryStore;
+  sampleLimit?: number;
+  pinsDir?: string;
+  checkpointsDir?: string;
+  workflowObservationsDir?: string;
+}
+
+interface FileLayerScanOptions {
+  dir: string;
+  layer: ScopeInventoryLayer;
+  sampleLimit: number;
+  readScope: (record: Record<string, unknown>) => unknown;
+  readId: (record: Record<string, unknown>, path: string) => string;
+  readContext: (record: Record<string, unknown>) => string;
+  readPreview: (record: Record<string, unknown>) => string;
+  readRecordedAt?: (record: Record<string, unknown>) => string | undefined;
+  recommendation: string;
+}
+
+function emptyCounts(): Record<LegacyScopeIssueKind, number> {
+  return {
+    missing: 0,
+    empty: 0,
+    global: 0,
+  };
+}
+
+function clampLimit(value: number | undefined, min: number, max: number, fallback: number): number {
+  if (!Number.isFinite(value)) return fallback;
+  return Math.min(max, Math.max(min, Math.floor(value as number)));
+}
+
+function clip(text: string, maxLen = 72): string {
+  const compact = text.replace(/\s+/g, " ").trim();
+  if (!compact) return "-";
+  return compact.length <= maxLen ? compact : `${compact.slice(0, maxLen - 3)}...`;
+}
+
+function formatLayerLabel(layer: ScopeInventoryLayer): string {
+  switch (layer) {
+    case "memories":
+      return "memories";
+    case "pins":
+      return "pins";
+    case "session-checkpoints":
+      return "session-checkpoints";
+    case "workflow-observations":
+      return "workflow-observations";
+  }
+}
+
+function normalizeRecordedAt(value: unknown): string | undefined {
+  if (typeof value !== "string") return undefined;
+  return Number.isFinite(Date.parse(value)) ? value : undefined;
+}
+
+function timestampToIso(value: number): string | undefined {
+  if (!Number.isFinite(value)) return undefined;
+  return new Date(value).toISOString();
+}
+
+function mapMemoryAudit(audit: LegacyScopeAudit): ScopeInventorySample[] {
+  return audit.samples.map((sample) => ({
+    layer: "memories",
+    id: sample.id,
+    kind: sample.kind,
+    scope: sample.scope,
+    context: sample.category,
+    preview: clip(sample.text, 80),
+    recordedAt: timestampToIso(sample.timestamp),
+  }));
+}
+
+function scanJsonLayer(options: FileLayerScanOptions): ScopeInventoryLayerReport {
+  const counts = emptyCounts();
+  if (!existsSync(options.dir)) {
+    return {
+      layer: options.layer,
+      scannedCount: 0,
+      anomalyCount: 0,
+      invalidCount: 0,
+      reviewedCount: 0,
+      counts,
+      samples: [],
+      recommendation: options.recommendation,
+    };
+  }
+
+  const files = readdirSync(options.dir)
+    .filter((name) => name.endsWith(".json"))
+    .map((name) => join(options.dir, name));
+
+  const anomalies: ScopeInventorySample[] = [];
+  let invalidCount = 0;
+  let reviewedCount = 0;
+
+  for (const path of files) {
+    let parsed: Record<string, unknown>;
+    try {
+      parsed = JSON.parse(readFileSync(path, "utf-8")) as Record<string, unknown>;
+    } catch {
+      invalidCount += 1;
+      continue;
+    }
+
+    const rawScope = options.readScope(parsed);
+    const kind = classifyLegacyScope(rawScope);
+    if (!kind) continue;
+    if (suppressesLegacyScopeIssue(parsed.legacyScopeReview, kind)) {
+      reviewedCount += 1;
+      continue;
+    }
+
+    counts[kind] += 1;
+    anomalies.push({
+      layer: options.layer,
+      id: options.readId(parsed, path),
+      kind,
+      scope: typeof rawScope === "string" ? rawScope : rawScope == null ? null : String(rawScope),
+      context: options.readContext(parsed),
+      preview: clip(options.readPreview(parsed), 80),
+      recordedAt: options.readRecordedAt?.(parsed),
+      sourcePath: path,
+    });
+  }
+
+  anomalies.sort((a, b) => {
+    const timeDiff = Date.parse(b.recordedAt || "") - Date.parse(a.recordedAt || "");
+    if (Number.isFinite(timeDiff) && timeDiff !== 0) return timeDiff;
+    return a.id.localeCompare(b.id);
+  });
+
+  return {
+    layer: options.layer,
+    scannedCount: files.length,
+    anomalyCount: counts.missing + counts.empty + counts.global,
+    invalidCount,
+    reviewedCount,
+    counts,
+    samples: anomalies.slice(0, options.sampleLimit),
+    recommendation: options.recommendation,
+  };
+}
+
+export async function collectScopeInventory(options: ScopeInventoryOptions): Promise<ScopeInventoryReport> {
+  const sampleLimit = clampLimit(options.sampleLimit, 1, 200, 20);
+  const [stats, audit] = await Promise.all([
+    options.store.stats(),
+    options.store.auditLegacyScopes(sampleLimit),
+  ]);
+
+  const memoryLayer: ScopeInventoryLayerReport = {
+    layer: "memories",
+    scannedCount: stats.totalCount,
+    anomalyCount: audit.totalCount,
+    invalidCount: 0,
+    reviewedCount: 0,
+    counts: audit.counts,
+    samples: mapMemoryAudit(audit),
+    recommendation: "Migrate or delete historical unscoped memory rows before removing compatibility shims.",
+  };
+
+  const pinsDir = options.pinsDir || resolve(import.meta.dir, "../data/pins");
+  const checkpointsDir = options.checkpointsDir || resolve(import.meta.dir, "../data/session-checkpoints");
+  const workflowObservationsDir = options.workflowObservationsDir || resolve(import.meta.dir, "../data/workflow-observations");
+
+  const pinLayer = scanJsonLayer({
+    dir: pinsDir,
+    layer: "pins",
+    sampleLimit,
+    readScope: (record) => (record.source as Record<string, unknown> | undefined)?.scope,
+    readId: (record, path) => typeof record.id === "string" ? record.id : basename(path, ".json"),
+    readContext: (record) => typeof record.type === "string" ? record.type : "pin",
+    readPreview: (record) => {
+      if (typeof record.title === "string" && record.title.trim()) return record.title;
+      if (typeof record.summary === "string" && record.summary.trim()) return record.summary;
+      return "Pinned asset with legacy source scope";
+    },
+    readRecordedAt: (record) =>
+      normalizeRecordedAt(record.updatedAt) || normalizeRecordedAt(record.createdAt),
+    recommendation: "Review pinned assets that still point at global/unscoped sources; re-pin them from a scoped memory if needed.",
+  });
+
+  const checkpointLayer = scanJsonLayer({
+    dir: checkpointsDir,
+    layer: "session-checkpoints",
+    sampleLimit,
+    readScope: (record) => record.resolvedScope,
+    readId: (record, path) => typeof record.checkpointId === "string" ? record.checkpointId : basename(path, ".json"),
+    readContext: (record) => typeof record.sessionId === "string" ? record.sessionId : "checkpoint",
+    readPreview: (record) => typeof record.summary === "string" ? record.summary : "Checkpoint with legacy scope state",
+    readRecordedAt: (record) => normalizeRecordedAt(record.updatedAt),
+    recommendation: "Review or delete legacy checkpoint files with missing/empty/global resolvedScope before relying on them for handoff context.",
+  });
+
+  const workflowLayer = scanJsonLayer({
+    dir: workflowObservationsDir,
+    layer: "workflow-observations",
+    sampleLimit,
+    readScope: (record) => record.resolvedScope,
+    readId: (record, path) => typeof record.observationId === "string" ? record.observationId : basename(path, ".json"),
+    readContext: (record) => typeof record.workflowId === "string" ? record.workflowId : "workflow",
+    readPreview: (record) => typeof record.summary === "string" ? record.summary : "Workflow observation with legacy scope state",
+    readRecordedAt: (record) => normalizeRecordedAt(record.recordedAt),
+    recommendation: "Review or delete legacy workflow observation files with missing/empty/global resolvedScope; they should stay outside normal continuity recall.",
+  });
+
+  const layers = [memoryLayer, pinLayer, checkpointLayer, workflowLayer];
+  return {
+    generatedAt: new Date().toISOString(),
+    sampleLimit,
+    totalScannedCount: layers.reduce((sum, layer) => sum + layer.scannedCount, 0),
+    totalAnomalyCount: layers.reduce((sum, layer) => sum + layer.anomalyCount, 0),
+    totalInvalidCount: layers.reduce((sum, layer) => sum + layer.invalidCount, 0),
+    totalReviewedCount: layers.reduce((sum, layer) => sum + layer.reviewedCount, 0),
+    layers,
+  };
+}
+
+function formatScope(value: string | null): string {
+  if (value == null) return "<missing>";
+  return value.trim() ? value.trim() : "<empty>";
+}
+
+export function formatScopeInventoryReport(report: ScopeInventoryReport): string {
+  const lines = [
+    "Scope inventory",
+    "",
+    `  Generated at   : ${report.generatedAt}`,
+    `  Records scanned: ${report.totalScannedCount}`,
+    `  Anomalies      : ${report.totalAnomalyCount}`,
+    `  Invalid files  : ${report.totalInvalidCount}`,
+    `  Reviewed keeps : ${report.totalReviewedCount}`,
+    "",
+    "  By layer:",
+  ];
+
+  for (const layer of report.layers) {
+    lines.push(
+      `    ${formatLayerLabel(layer.layer).padEnd(22)} scanned ${String(layer.scannedCount).padStart(4)}  anomalies ${String(layer.anomalyCount).padStart(3)}  invalid ${String(layer.invalidCount).padStart(3)}  reviewed ${String(layer.reviewedCount).padStart(3)}  missing ${String(layer.counts.missing).padStart(3)}  empty ${String(layer.counts.empty).padStart(3)}  global ${String(layer.counts.global).padStart(3)}`
+    );
+  }
+
+  const anomalyLayers = report.layers.filter((layer) => layer.samples.length > 0);
+  if (anomalyLayers.length === 0) {
+    if (report.totalInvalidCount === 0) {
+      lines.push("", "No unresolved legacy scope anomalies found across memories, pins, checkpoints, or workflow observations.");
+    }
+    return lines.join("\n");
+  }
+
+  for (const layer of anomalyLayers) {
+    lines.push("", `  [${formatLayerLabel(layer.layer)}]`);
+    lines.push("    ID        Kind     Scope        Context             Date        Preview");
+    lines.push("    --------  -------  -----------  ------------------  ----------  -------");
+    for (const sample of layer.samples) {
+      lines.push(
+        `    ${sample.id.slice(0, 8).padEnd(8)}  ${sample.kind.padEnd(7)}  ${formatScope(sample.scope).padEnd(11)}  ${clip(sample.context, 18).padEnd(18)}  ${(sample.recordedAt || "-").slice(0, 10).padEnd(10)}  ${clip(sample.preview, 56)}`
+      );
+    }
+    lines.push(`    Next: ${layer.recommendation}`);
+  }
+
+  return lines.join("\n");
+}

package/src/scope-policy.ts

@@ -0,0 +1,142 @@
+import type { RetrievalContext } from "./retriever.js";
+
+function normalizeScopeValue(value: string | undefined): string | undefined {
+  const trimmed = typeof value === "string" ? value.replace(/\s+/g, " ").trim() : "";
+  return trimmed || undefined;
+}
+
+function envScopeCandidate(env: NodeJS.ProcessEnv): { scope?: string; inferredFrom?: string } {
+  const explicitEnvKeys = [
+    "RECALLNEST_DEFAULT_SCOPE",
+    "RECALLNEST_SCOPE",
+    "RECALLNEST_PROJECT_SCOPE",
+  ] as const;
+
+  for (const key of explicitEnvKeys) {
+    const value = normalizeScopeValue(env[key]);
+    if (value) {
+      return {
+        scope: value,
+        inferredFrom: key,
+      };
+    }
+  }
+
+  const sessionId = normalizeScopeValue(env.RECALLNEST_SESSION_ID);
+  if (sessionId) {
+    return {
+      scope: `session:${sessionId}`,
+      inferredFrom: "RECALLNEST_SESSION_ID",
+    };
+  }
+
+  return {};
+}
+
+export function resolveSessionScope(sessionId?: string): string | undefined {
+  const normalized = normalizeScopeValue(sessionId);
+  return normalized ? `session:${normalized}` : undefined;
+}
+
+export function matchesScopeFilter(rowScope: string, scopeFilter?: string[]): boolean {
+  if (!scopeFilter || scopeFilter.length === 0) return true;
+  return scopeFilter.some((scope) => scope.includes(":") ? rowScope === scope : rowScope.startsWith(scope));
+}
+
+export interface ScopeSelectionOptions {
+  scope?: string;
+  sessionId?: string;
+  allScopes?: boolean;
+  operation: string;
+  env?: NodeJS.ProcessEnv;
+  allowUnscoped?: boolean;
+}
+
+export interface ScopeSelection {
+  allScopes: boolean;
+  resolvedScope?: string;
+  scopeFilter?: string[];
+  inferredFrom?: string;
+}
+
+export function resolveScopeSelection(options: ScopeSelectionOptions): ScopeSelection {
+  if (options.allScopes) {
+    return {
+      allScopes: true,
+      scopeFilter: undefined,
+      inferredFrom: "allScopes",
+    };
+  }
+
+  const explicitScope = normalizeScopeValue(options.scope);
+  if (explicitScope) {
+    return {
+      allScopes: false,
+      resolvedScope: explicitScope,
+      scopeFilter: [explicitScope],
+      inferredFrom: "scope",
+    };
+  }
+
+  const sessionScope = resolveSessionScope(options.sessionId);
+  if (sessionScope) {
+    return {
+      allScopes: false,
+      resolvedScope: sessionScope,
+      scopeFilter: [sessionScope],
+      inferredFrom: "sessionId",
+    };
+  }
+
+  const envSelection = envScopeCandidate(options.env || process.env);
+  if (envSelection.scope) {
+    return {
+      allScopes: false,
+      resolvedScope: envSelection.scope,
+      scopeFilter: [envSelection.scope],
+      inferredFrom: envSelection.inferredFrom,
+    };
+  }
+
+  if (options.allowUnscoped) {
+    return {
+      allScopes: false,
+      scopeFilter: undefined,
+    };
+  }
+
+  throw new Error(
+    `${options.operation} requires a scope. Pass scope explicitly, provide sessionId, or set ` +
+    `RECALLNEST_DEFAULT_SCOPE / RECALLNEST_SCOPE / RECALLNEST_SESSION_ID. ` +
+    `Use allScopes=true only for explicit cross-scope reads.`,
+  );
+}
+
+export function buildRetrievalContext(
+  base: Omit<RetrievalContext, "scopeFilter"> & {
+    scope?: string;
+    sessionId?: string;
+    allScopes?: boolean;
+  },
+  options: Pick<ScopeSelectionOptions, "operation" | "env" | "allowUnscoped">,
+): RetrievalContext {
+  const selection = resolveScopeSelection({
+    scope: base.scope,
+    sessionId: base.sessionId,
+    allScopes: base.allScopes,
+    operation: options.operation,
+    env: options.env,
+    allowUnscoped: options.allowUnscoped,
+  });
+
+  return {
+    query: base.query,
+    limit: base.limit,
+    category: base.category,
+    source: base.source,
+    includeArchived: base.includeArchived,
+    trace: base.trace,
+    graph: base.graph,
+    scopeFilter: selection.scopeFilter,
+  };
+}

package/src/session-engine.ts

@@ -0,0 +1,168 @@
+import { randomUUID } from "node:crypto";
+
+import {
+  type SessionCheckpointInput,
+  SessionCheckpointInputSchema,
+  type SessionCheckpointRecord,
+  SessionCheckpointRecordSchema,
+} from "./session-schema.js";
+
+const REPO_STATE_PATTERN = /\bgit status\b|未提交|staged|uncommitted|已修改文件|modified files?|untracked|新增文件|dirty repo|dirty worktree/iu;
+const SENTENCE_SPLIT_PATTERN = /(?:\r?\n)+|(?<=[。！？!?;；])\s+/u;
+const CHECKPOINT_SANITIZABLE_FIELDS = ["summary", "task", "decisions", "openLoops", "nextActions"] as const;
+
+type CheckpointSanitizableField = typeof CHECKPOINT_SANITIZABLE_FIELDS[number];
+
+interface SanitizedStringResult {
+  value: string;
+  changed: boolean;
+}
+
+interface SanitizedListResult {
+  value: string[];
+  changed: boolean;
+}
+
+export interface SessionCheckpointSanitizationReport {
+  changed: boolean;
+  changedFields: CheckpointSanitizableField[];
+}
+
+export type CheckpointQuality = "rich" | "minimal";
+
+export interface SessionCheckpointBuildResult {
+  record: SessionCheckpointRecord;
+  sanitization: SessionCheckpointSanitizationReport;
+  quality: CheckpointQuality;
+}
+
+function hasRepoStateText(text: string): boolean {
+  return REPO_STATE_PATTERN.test(text);
+}
+
+function normalizeCheckpointText(text: string): string {
+  return text.replace(/\s+/g, " ").trim();
+}
+
+function sanitizeCheckpointSummary(summary: string): SanitizedStringResult {
+  const parts = summary
+    .split(SENTENCE_SPLIT_PATTERN)
+    .map(normalizeCheckpointText)
+    .filter(Boolean);
+  const kept = parts.filter((part) => !hasRepoStateText(part));
+  const value = kept[0] ? kept.join(" ") : "Checkpoint captured current task state without repo-state details.";
+  return {
+    value,
+    changed: kept.length !== parts.length,
+  };
+}
+
+function sanitizeCheckpointList(items: string[], fallbackWhenRemoved?: string): SanitizedListResult {
+  const kept = items
+    .map(normalizeCheckpointText)
+    .filter(Boolean)
+    .filter((item) => !hasRepoStateText(item));
+
+  if (kept.length > 0) {
+    return {
+      value: kept,
+      changed: kept.length !== items.length,
+    };
+  }
+  if (fallbackWhenRemoved && items.some((item) => hasRepoStateText(String(item)))) {
+    return {
+      value: [fallbackWhenRemoved],
+      changed: true,
+    };
+  }
+  return {
+    value: kept,
+    changed: kept.length !== items.length,
+  };
+}
+
+/**
+ * Normalize checkpoint scope to a canonical form:
+ * - lowercase
+ * - bare project names (e.g. "recallnest", "RecallNest") get "project:" prefix
+ * - session:* and project:* prefixes are preserved
+ */
+export function normalizeCheckpointScope(scope: string): string {
+  const trimmed = scope.trim().toLowerCase();
+  if (trimmed.startsWith("session:") || trimmed.startsWith("project:")) return trimmed;
+  if (trimmed.startsWith("memory:") || trimmed.startsWith("eval:") || trimmed.startsWith("asset:")) return trimmed;
+  // Bare project name — add project: prefix if it looks like an identifier (no spaces, no colons)
+  if (/^[a-z0-9_-]+$/i.test(trimmed)) return `project:${trimmed}`;
+  return trimmed;
+}
+
+export function resolveCheckpointScope(input: Pick<SessionCheckpointInput, "sessionId" | "scope">): string {
+  const raw = input.scope || `session:${input.sessionId}`;
+  return normalizeCheckpointScope(raw);
+}
+
+export function buildSessionCheckpointResult(rawInput: unknown): SessionCheckpointBuildResult {
+  const input = SessionCheckpointInputSchema.parse(rawInput);
+  const summary = sanitizeCheckpointSummary(input.summary);
+  const taskChanged = Boolean(input.task && hasRepoStateText(input.task));
+  const decisions = sanitizeCheckpointList(input.decisions);
+  const openLoops = sanitizeCheckpointList(
+    input.openLoops,
+    "Current repo state still needs local verification if it matters for the next task.",
+  );
+  const nextActions = sanitizeCheckpointList(
+    input.nextActions,
+    "Verify current repo state locally if it matters for the next task.",
+  );
+  const sanitizedInput: SessionCheckpointInput = {
+    ...input,
+    summary: summary.value,
+    task: taskChanged ? undefined : input.task,
+    decisions: decisions.value,
+    openLoops: openLoops.value,
+    nextActions: nextActions.value,
+  };
+  const changedFields = CHECKPOINT_SANITIZABLE_FIELDS.filter((field) => {
+    switch (field) {
+      case "summary":
+        return summary.changed;
+      case "task":
+        return taskChanged;
+      case "decisions":
+        return decisions.changed;
+      case "openLoops":
+        return openLoops.changed;
+      case "nextActions":
+        return nextActions.changed;
+      default:
+        return false;
+    }
+  });
+
+  const record = SessionCheckpointRecordSchema.parse({
+    ...sanitizedInput,
+    checkpointId: randomUUID(),
+    resolvedScope: resolveCheckpointScope(sanitizedInput),
+  });
+
+  const FALLBACK_SUMMARY = "Checkpoint captured current task state without repo-state details.";
+  const isFallbackSummary = record.summary === FALLBACK_SUMMARY;
+  const hasStructuredContent =
+    record.decisions.length > 0 ||
+    record.openLoops.length > 0 ||
+    record.nextActions.length > 0;
+  const quality: CheckpointQuality = isFallbackSummary && !hasStructuredContent ? "minimal" : "rich";
+
+  return {
+    record,
+    sanitization: {
+      changed: changedFields.length > 0,
+      changedFields,
+    },
+    quality,
+  };
+}
+
+export function buildSessionCheckpointRecord(rawInput: unknown): SessionCheckpointRecord {
+  return buildSessionCheckpointResult(rawInput).record;
+}