pi-memory-stone 0.1.3 → 0.1.5

package/src/vault/index.ts

@@ -0,0 +1,306 @@
+/**
+ * Optional Obsidian-compatible knowledge vault layer for pi-memory-stone.
+ *
+ * SQLite remains the source of truth. Vault pages are generated, reviewable
+ * markdown projections of active memory records.
+ */
+
+import { createHash } from "node:crypto";
+import { existsSync, mkdirSync, readFileSync, readdirSync, statSync, writeFileSync } from "node:fs";
+import { join, relative } from "node:path";
+import { listRecords, type RecordRow } from "../db/index.js";
+import { kindDirectory, recordMarkdown, recordTitle } from "./markdown.js";
+import { resolveVaultPath, type VaultScope } from "./paths.js";
+
+export { isVaultScope, parseVaultScope, resolveVaultPath, type VaultScope } from "./paths.js";
+export { kindDirectory, parseTags, recordMarkdown, recordTitle, sanitizeSlug } from "./markdown.js";
+
+const VAULT_SCHEMA_VERSION = 1;
+
+export interface VaultInitResult {
+  path: string;
+  created: boolean;
+}
+
+export interface VaultSyncResult {
+  path: string;
+  records: number;
+  pagesWritten: number;
+  registryPath: string;
+}
+
+export interface VaultStatus {
+  path: string;
+  initialized: boolean;
+  registryExists: boolean;
+  pageCount: number;
+  recordPageCount: number;
+  lastSyncedAt: string | null;
+}
+
+export interface VaultRegistry {
+  format: "pi-memory-stone-vault-registry";
+  version: number;
+  scope: VaultScope;
+  project_id: string | null;
+  generated_at: string;
+  pages: VaultRegistryPage[];
+}
+
+export interface VaultRegistryPage {
+  path: string;
+  title: string;
+  kind: string;
+  source_record_id?: string;
+  source_url?: string;
+  source_packet?: string;
+  content_hash: string;
+  generated: true;
+  created_at: string;
+  updated_at: string;
+}
+
+export function initVault(scope: VaultScope, projectId: string | null, cwd: string): VaultInitResult {
+  const vaultPath = resolveVaultPath(scope, projectId, cwd);
+  const alreadyInitialized = existsSync(join(vaultPath, "WIKI_SCHEMA.md"));
+
+  ensureVaultDirectories(vaultPath);
+  writeIfMissing(join(vaultPath, "WIKI_SCHEMA.md"), schemaMarkdown(scope));
+  writeIfMissing(join(vaultPath, "index.md"), indexMarkdown(scope, projectId, []));
+  writeIfMissing(join(vaultPath, "meta", "registry.json"), JSON.stringify(emptyRegistry(scope, projectId), null, 2) + "\n");
+
+  return { path: vaultPath, created: !alreadyInitialized };
+}
+
+export function syncVault(scope: VaultScope, projectId: string | null, cwd: string): VaultSyncResult {
+  const vaultPath = resolveVaultPath(scope, projectId, cwd);
+  if (!isVaultInitialized(vaultPath)) {
+    throw new Error(`Vault is not initialized at ${vaultPath}. Run /memory-vault-init first.`);
+  }
+
+  ensureVaultDirectories(vaultPath);
+  const records = recordsForVault(scope, projectId);
+  const pages: VaultRegistryPage[] = [];
+  let pagesWritten = 0;
+
+  for (const record of records) {
+    const relativePagePath = join("records", kindDirectory(record.kind), `${record.id}.md`);
+    const outputPath = join(vaultPath, relativePagePath);
+    mkdirSync(join(vaultPath, "records", kindDirectory(record.kind)), { recursive: true, mode: 0o700 });
+
+    const content = recordMarkdown(record);
+    const hash = sha256(content);
+    const existing = existsSync(outputPath) ? readFileSync(outputPath, "utf8") : null;
+    if (existing !== content) {
+      writeFileSync(outputPath, content, { mode: 0o600 });
+      pagesWritten += 1;
+    }
+
+    pages.push({
+      path: normalizePath(relativePagePath),
+      title: recordTitle(record),
+      kind: record.kind,
+      source_record_id: record.id,
+      content_hash: hash,
+      generated: true,
+      created_at: new Date(record.created_at).toISOString(),
+      updated_at: new Date(record.updated_at).toISOString(),
+    });
+  }
+
+  const existingRegistry = readRegistry(join(vaultPath, "meta", "registry.json"));
+  const preservedPages = existingRegistry?.pages.filter((page) => !page.source_record_id) ?? [];
+  const registry: VaultRegistry = {
+    format: "pi-memory-stone-vault-registry",
+    version: VAULT_SCHEMA_VERSION,
+    scope,
+    project_id: scope === "project" ? projectId : null,
+    generated_at: new Date().toISOString(),
+    pages: [...preservedPages, ...pages].sort((a, b) => a.path.localeCompare(b.path)),
+  };
+
+  const registryPath = join(vaultPath, "meta", "registry.json");
+  writeFileSync(registryPath, JSON.stringify(registry, null, 2) + "\n", { mode: 0o600 });
+  writeFileSync(join(vaultPath, "index.md"), indexMarkdown(scope, projectId, pages), { mode: 0o600 });
+
+  return { path: vaultPath, records: records.length, pagesWritten, registryPath };
+}
+
+export function getVaultStatus(scope: VaultScope, projectId: string | null, cwd: string): VaultStatus {
+  const vaultPath = resolveVaultPath(scope, projectId, cwd);
+  const registryPath = join(vaultPath, "meta", "registry.json");
+  const initialized = isVaultInitialized(vaultPath);
+  const registry = readRegistry(registryPath);
+
+  return {
+    path: vaultPath,
+    initialized,
+    registryExists: existsSync(registryPath),
+    pageCount: countMarkdownFiles(vaultPath),
+    recordPageCount: registry?.pages.filter((page) => Boolean(page.source_record_id)).length ?? 0,
+    lastSyncedAt: registry?.generated_at ?? null,
+  };
+}
+
+function recordsForVault(scope: VaultScope, projectId: string | null): RecordRow[] {
+  return listRecords().filter((record) => {
+    if (scope === "personal") return record.scope === "global";
+    return record.scope === "project" && record.project_id === projectId;
+  });
+}
+
+function ensureVaultDirectories(vaultPath: string): void {
+  const dirs = [
+    vaultPath,
+    join(vaultPath, "records"),
+    join(vaultPath, "records", "decisions"),
+    join(vaultPath, "records", "preferences"),
+    join(vaultPath, "records", "tasks"),
+    join(vaultPath, "records", "error-resolutions"),
+    join(vaultPath, "records", "turn-summaries"),
+    join(vaultPath, "records", "session-summaries"),
+    join(vaultPath, "records", "file-activity"),
+    join(vaultPath, "concepts"),
+    join(vaultPath, "projects"),
+    join(vaultPath, "sessions"),
+    join(vaultPath, "syntheses"),
+    join(vaultPath, "sources"),
+    join(vaultPath, "meta"),
+  ];
+  for (const dir of dirs) {
+    mkdirSync(dir, { recursive: true, mode: 0o700 });
+  }
+}
+
+function isVaultInitialized(vaultPath: string): boolean {
+  return existsSync(join(vaultPath, "WIKI_SCHEMA.md")) && existsSync(join(vaultPath, "meta", "registry.json"));
+}
+
+function writeIfMissing(path: string, content: string): void {
+  if (!existsSync(path)) {
+    writeFileSync(path, content, { mode: 0o600 });
+  }
+}
+
+function emptyRegistry(scope: VaultScope, projectId: string | null): VaultRegistry {
+  return {
+    format: "pi-memory-stone-vault-registry",
+    version: VAULT_SCHEMA_VERSION,
+    scope,
+    project_id: scope === "project" ? projectId : null,
+    generated_at: new Date().toISOString(),
+    pages: [],
+  };
+}
+
+function schemaMarkdown(scope: VaultScope): string {
+  return [
+    "# Memory Stone Vault Schema",
+    "",
+    "This vault is an Obsidian-compatible markdown projection of pi-memory-stone records.",
+    "SQLite remains the source of truth; generated pages may be overwritten by `/memory-vault-sync`.",
+    "",
+    "## Layout",
+    "",
+    "```txt",
+    ".memory-stone/vault/ or ~/.pi/agent/memory/vaults/personal/",
+    "  index.md",
+    "  records/",
+    "    decisions/",
+    "    preferences/",
+    "    tasks/",
+    "    error-resolutions/",
+    "    turn-summaries/",
+    "    session-summaries/",
+    "  concepts/",
+    "  projects/",
+    "  sessions/",
+    "  syntheses/",
+    "  sources/",
+    "  meta/registry.json",
+    "```",
+    "",
+    "## Scope",
+    "",
+    `This vault was initialized as a \`${scope}\` vault.`,
+    "",
+    "## Generated pages",
+    "",
+    "Generated pages include frontmatter with `generated: true` and `source: pi-memory-stone`.",
+    "Use human-authored pages outside `records/` or clear `generated` before treating edits as durable.",
+    "",
+  ].join("\n");
+}
+
+function indexMarkdown(scope: VaultScope, projectId: string | null, pages: VaultRegistryPage[]): string {
+  const byKind = new Map<string, number>();
+  for (const page of pages) {
+    byKind.set(page.kind, (byKind.get(page.kind) ?? 0) + 1);
+  }
+
+  const lines: string[] = [];
+  lines.push("# Memory Stone Vault");
+  lines.push("");
+  lines.push(`Scope: \`${scope}\``);
+  if (projectId && scope === "project") lines.push(`Project: \`${projectId}\``);
+  lines.push(`Generated pages: ${pages.length}`);
+  lines.push("");
+  lines.push("## Records by kind");
+  lines.push("");
+  if (byKind.size === 0) {
+    lines.push("_No synced records yet._");
+  } else {
+    for (const [kind, count] of [...byKind.entries()].sort(([a], [b]) => a.localeCompare(b))) {
+      lines.push(`- [[${kind}]]: ${count}`);
+    }
+  }
+  lines.push("");
+  lines.push("## Generated record pages");
+  lines.push("");
+  for (const page of pages) {
+    lines.push(`- [${page.title}](${encodeURI(page.path)})`);
+  }
+  lines.push("");
+  return lines.join("\n");
+}
+
+function readRegistry(path: string): VaultRegistry | null {
+  if (!existsSync(path)) return null;
+  try {
+    const parsed = JSON.parse(readFileSync(path, "utf8")) as VaultRegistry;
+    return parsed?.format === "pi-memory-stone-vault-registry" ? parsed : null;
+  } catch {
+    return null;
+  }
+}
+
+function countMarkdownFiles(root: string): number {
+  if (!existsSync(root)) return 0;
+  let count = 0;
+  const stack = [root];
+  while (stack.length > 0) {
+    const dir = stack.pop()!;
+    for (const name of readdirSync(dir)) {
+      const path = join(dir, name);
+      const stat = statSync(path);
+      if (stat.isDirectory()) {
+        stack.push(path);
+      } else if (stat.isFile() && path.endsWith(".md")) {
+        count += 1;
+      }
+    }
+  }
+  return count;
+}
+
+function normalizePath(path: string): string {
+  return path.split(/[\\/]+/).join("/");
+}
+
+function sha256(content: string): string {
+  return createHash("sha256").update(content).digest("hex").slice(0, 16);
+}
+
+export function relativeVaultPath(vaultPath: string, filePath: string): string {
+  return normalizePath(relative(vaultPath, filePath));
+}

package/src/vault/intent.ts

@@ -0,0 +1,37 @@
+/** Natural-language intent parsing for lightweight vault capture requests. */
+
+import { isVaultScope, type VaultScope } from "./paths.js";
+
+const URL_PATTERN = /https?:\/\/[^\s<>)\]"']+/i;
+
+export interface VaultCaptureIntent {
+  url: string;
+  scope: VaultScope;
+}
+
+export function parseVaultCaptureIntent(prompt: string): VaultCaptureIntent | null {
+  const url = extractFirstUrl(prompt);
+  if (!url) return null;
+
+  const lower = prompt.toLowerCase();
+  if (!lower.includes("vault")) return null;
+  if (!/\b(add|capture|save|store|clip|archive|ingest)\b/.test(lower)) return null;
+
+  return {
+    url,
+    scope: inferScope(lower),
+  };
+}
+
+export function extractFirstUrl(text: string): string | null {
+  const match = text.match(URL_PATTERN);
+  if (!match) return null;
+  return match[0].replace(/[.,;:!?]+$/, "");
+}
+
+function inferScope(lowerPrompt: string): VaultScope {
+  const explicit = lowerPrompt.match(/--scope\s+(project|personal)\b/)?.[1];
+  if (explicit && isVaultScope(explicit)) return explicit;
+  if (/--personal\b|\bpersonal(?:\s+memory)?\s+vault\b|\bglobal(?:\s+memory)?\s+vault\b/.test(lowerPrompt)) return "personal";
+  return "project";
+}

package/src/vault/markdown.ts

@@ -0,0 +1,120 @@
+/** Markdown rendering helpers for Obsidian-compatible memory vault pages. */
+
+import type { RecordRow } from "../db/index.js";
+
+const KIND_TITLES: Record<string, string> = {
+  decision: "Decision",
+  preference: "Preference",
+  task: "Task",
+  error_resolution: "Error Resolution",
+  turn_summary: "Turn Summary",
+  session_summary: "Session Summary",
+  file_activity: "File Activity",
+};
+
+export function recordTitle(record: RecordRow): string {
+  const prefix = KIND_TITLES[record.kind] ?? record.kind;
+  const firstLine = record.text.split(/\r?\n/).map((line) => line.trim()).find(Boolean) ?? record.id;
+  const cleaned = firstLine.replace(/^#+\s*/, "").replace(/\s+/g, " ").trim();
+  const excerpt = cleaned.length > 80 ? `${cleaned.slice(0, 77)}...` : cleaned;
+  return `${prefix}: ${excerpt}`;
+}
+
+export function kindDirectory(kind: string): string {
+  switch (kind) {
+    case "decision": return "decisions";
+    case "preference": return "preferences";
+    case "task": return "tasks";
+    case "error_resolution": return "error-resolutions";
+    case "turn_summary": return "turn-summaries";
+    case "session_summary": return "session-summaries";
+    case "file_activity": return "file-activity";
+    default: return sanitizeSlug(kind);
+  }
+}
+
+export function recordMarkdown(record: RecordRow): string {
+  const title = recordTitle(record);
+  const tags = parseTags(record.tags);
+  const frontmatter = renderFrontmatter({
+    id: record.id,
+    kind: record.kind,
+    scope: record.scope,
+    project_id: record.project_id,
+    session_id: record.session_id,
+    created_at: new Date(record.created_at).toISOString(),
+    updated_at: new Date(record.updated_at).toISOString(),
+    tags,
+    generated: true,
+    source: "pi-memory-stone",
+  });
+
+  const lines: string[] = [];
+  lines.push(frontmatter);
+  lines.push(`# ${escapeMarkdownHeading(title)}`);
+  lines.push("");
+  lines.push(record.text.trim() || "_No memory text captured._");
+  lines.push("");
+  lines.push("## Metadata");
+  lines.push("");
+  lines.push(`- Memory ref: \`${record.id}\``);
+  lines.push(`- Kind: \`${record.kind}\``);
+  lines.push(`- Scope: \`${record.scope}\``);
+  if (record.project_id) lines.push(`- Project: \`${record.project_id}\``);
+  if (record.session_id) lines.push(`- Session: \`${record.session_id}\``);
+  if (tags.length > 0) lines.push(`- Tags: ${tags.map((tag) => `#${sanitizeTag(tag)}`).join(" ")}`);
+  lines.push("");
+  lines.push("## Links");
+  lines.push("");
+  lines.push(`- [[${record.kind}]]`);
+  lines.push("- [[pi-memory-stone]]");
+  lines.push("");
+
+  return lines.join("\n");
+}
+
+export function parseTags(tags: string | null | undefined): string[] {
+  if (!tags) return [];
+  return tags
+    .split(/[;,]/)
+    .map((tag) => tag.trim())
+    .filter(Boolean);
+}
+
+export function sanitizeSlug(value: string): string {
+  const slug = value
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-+|-+$/g, "");
+  return slug || "untitled";
+}
+
+function sanitizeTag(tag: string): string {
+  return tag.replace(/\s+/g, "-").replace(/[^\p{L}\p{N}_/-]/gu, "");
+}
+
+function escapeMarkdownHeading(value: string): string {
+  return value.replace(/[\r\n]+/g, " ").trim();
+}
+
+type FrontmatterValue = string | number | boolean | null | string[];
+
+function renderFrontmatter(values: Record<string, FrontmatterValue>): string {
+  const lines = ["---"];
+  for (const [key, value] of Object.entries(values)) {
+    if (value === null) continue;
+    if (Array.isArray(value)) {
+      lines.push(`${key}: [${value.map((item) => yamlString(item)).join(", ")}]`);
+    } else if (typeof value === "boolean" || typeof value === "number") {
+      lines.push(`${key}: ${value}`);
+    } else {
+      lines.push(`${key}: ${yamlString(value)}`);
+    }
+  }
+  lines.push("---");
+  return lines.join("\n");
+}
+
+function yamlString(value: string): string {
+  return JSON.stringify(value);
+}

package/src/vault/paths.ts

@@ -0,0 +1,44 @@
+/**
+ * Knowledge vault path resolution.
+ *
+ * Project vaults are opt-in and live inside the current project. Personal
+ * vaults stay under pi-memory-stone's private memory directory by default.
+ */
+
+import { join } from "node:path";
+import { getMemoryDir } from "../config/index.js";
+
+export type VaultScope = "project" | "personal";
+
+export function resolveVaultPath(scope: VaultScope, projectId: string | null, cwd: string): string {
+  if (scope === "personal") {
+    return process.env.PI_MEMORY_STONE_PERSONAL_VAULT_PATH
+      ?? join(getMemoryDir(), "vaults", "personal");
+  }
+
+  const root = projectId ?? cwd;
+  return join(root, ".memory-stone", "vault");
+}
+
+export function resolveSourcePacketPath(scope: VaultScope, projectId: string | null, cwd: string, captureId: string): string {
+  if (scope === "personal") {
+    return join(getMemoryDir(), "source-packets", "personal", captureId);
+  }
+
+  const root = projectId ?? cwd;
+  return join(root, ".memory-stone", "source-packets", captureId);
+}
+
+export function isVaultScope(value: string): value is VaultScope {
+  return value === "project" || value === "personal";
+}
+
+export function parseVaultScope(args: { flags: Set<string>; options: Map<string, string> }): VaultScope | undefined {
+  const explicitScope = args.options.get("scope");
+  if (explicitScope) {
+    return isVaultScope(explicitScope) ? explicitScope : undefined;
+  }
+  if (args.flags.has("personal")) return "personal";
+  if (args.flags.has("project")) return "project";
+  return "project";
+}

package/src/vault/quality.ts

@@ -0,0 +1,65 @@
+/** Extraction quality scoring for captured source pages. */
+
+export type CaptureQuality = "good" | "weak";
+
+export interface CaptureQualityReport {
+  quality: CaptureQuality;
+  score: number;
+  warnings: string[];
+  plainTextChars: number;
+}
+
+export function assessCaptureQuality(input: { title: string; markdown: string; extractor: string }): CaptureQualityReport {
+  const plain = markdownToPlainText(input.markdown);
+  const warnings: string[] = [];
+  let score = 0;
+
+  if (input.title.trim().length > 0) score += 0.2;
+  else warnings.push("missing title");
+
+  if (plain.length >= 500) score += 0.35;
+  else if (plain.length >= 120) {
+    score += 0.18;
+    warnings.push("short extracted text");
+  } else {
+    warnings.push("very short extracted text");
+  }
+
+  const paragraphCount = input.markdown.split(/\n{2,}/).filter((paragraph) => markdownToPlainText(paragraph).length >= 40).length;
+  if (paragraphCount >= 3) score += 0.2;
+  else warnings.push("few article-like paragraphs");
+
+  const linkOnlyRatio = linkOnlyLineRatio(input.markdown);
+  if (linkOnlyRatio <= 0.35) score += 0.15;
+  else warnings.push("high link/navigation ratio");
+
+  if (input.extractor === "html-readability" || input.extractor === "markdown" || input.extractor === "text") {
+    score += 0.1;
+  }
+
+  score = Math.min(1, Number(score.toFixed(2)));
+  return {
+    quality: score >= 0.65 ? "good" : "weak",
+    score,
+    warnings,
+    plainTextChars: plain.length,
+  };
+}
+
+function markdownToPlainText(markdown: string): string {
+  return markdown
+    .replace(/```[\s\S]*?```/g, " ")
+    .replace(/`[^`]*`/g, " ")
+    .replace(/!\[[^\]]*]\([^)]*\)/g, " ")
+    .replace(/\[([^\]]+)]\([^)]*\)/g, "$1")
+    .replace(/[#>*_~\-]+/g, " ")
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function linkOnlyLineRatio(markdown: string): number {
+  const lines = markdown.split(/\r?\n/).map((line) => line.trim()).filter(Boolean);
+  if (lines.length === 0) return 1;
+  const linkish = lines.filter((line) => /^[-*]?\s*\[[^\]]+][^)]+\)?\s*$/.test(line) || /^[-*]?\s*https?:\/\//.test(line)).length;
+  return linkish / lines.length;
+}

package/src/vault/url-resolvers.ts

@@ -0,0 +1,113 @@
+/** URL normalization and article-capture candidate resolution. */
+
+export type CaptureCandidateKind = "html" | "markdown" | "text" | "pdf" | "unknown";
+
+export interface CaptureCandidate {
+  url: string;
+  kind: CaptureCandidateKind;
+  strategy: string;
+  priority: number;
+}
+
+export interface CaptureTargets {
+  originalUrl: string;
+  canonicalUrl: string;
+  candidates: CaptureCandidate[];
+}
+
+export function resolveCaptureTargets(inputUrl: string): CaptureTargets {
+  const parsed = parseHttpUrl(inputUrl);
+  const candidates: CaptureCandidate[] = [];
+
+  const gistRaw = resolveGistRaw(parsed);
+  if (gistRaw) {
+    candidates.push({ url: gistRaw, kind: "markdown", strategy: "gist-raw", priority: 100 });
+  }
+
+  const githubRaw = resolveGithubRaw(parsed);
+  if (githubRaw) {
+    candidates.push({ url: githubRaw.url, kind: githubRaw.kind, strategy: "github-raw", priority: 95 });
+  }
+
+  candidates.push({
+    url: parsed.href,
+    kind: inferKind(parsed.href),
+    strategy: "direct",
+    priority: 10,
+  });
+
+  const deduped = dedupeCandidates(candidates)
+    .sort((a, b) => b.priority - a.priority);
+
+  return {
+    originalUrl: parsed.href,
+    canonicalUrl: deduped[0]?.url ?? parsed.href,
+    candidates: deduped,
+  };
+}
+
+export function parseHttpUrl(url: string): URL {
+  let parsed: URL;
+  try {
+    parsed = new URL(url);
+  } catch {
+    throw new Error(`Invalid URL: ${url}`);
+  }
+  if (parsed.protocol !== "http:" && parsed.protocol !== "https:") {
+    throw new Error("Only http:// and https:// URLs can be captured");
+  }
+  return parsed;
+}
+
+function resolveGistRaw(url: URL): string | null {
+  if (url.hostname === "gist.githubusercontent.com" && url.pathname.includes("/raw")) {
+    return url.href;
+  }
+  if (url.hostname !== "gist.github.com") return null;
+
+  const parts = url.pathname.split("/").filter(Boolean);
+  if (parts.length < 2) return null;
+
+  const [owner, gistId] = parts;
+  if (!owner || !gistId) return null;
+
+  return `https://gist.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(gistId)}/raw`;
+}
+
+function resolveGithubRaw(url: URL): { url: string; kind: CaptureCandidateKind } | null {
+  if (url.hostname === "raw.githubusercontent.com") {
+    return { url: url.href, kind: inferKind(url.href) };
+  }
+  if (url.hostname !== "github.com") return null;
+
+  const parts = url.pathname.split("/").filter(Boolean);
+  if (parts.length < 5) return null;
+
+  const [owner, repo, mode, branch, ...fileParts] = parts;
+  if (!owner || !repo || !branch || fileParts.length === 0) return null;
+  if (mode !== "blob" && mode !== "raw") return null;
+
+  const rawUrl = new URL(`https://raw.githubusercontent.com/${encodeURIComponent(owner)}/${encodeURIComponent(repo)}/${encodeURIComponent(branch)}/${fileParts.map(encodeURIComponent).join("/")}`);
+  rawUrl.search = url.search;
+  return { url: rawUrl.href, kind: inferKind(rawUrl.href) };
+}
+
+function inferKind(url: string): CaptureCandidateKind {
+  const path = new URL(url).pathname.toLowerCase();
+  if (path.endsWith(".md") || path.endsWith(".markdown") || path.endsWith(".mdx")) return "markdown";
+  if (path.endsWith(".txt") || path.endsWith(".text")) return "text";
+  if (path.endsWith(".pdf")) return "pdf";
+  if (path.endsWith(".html") || path.endsWith(".htm")) return "html";
+  return "unknown";
+}
+
+function dedupeCandidates(candidates: CaptureCandidate[]): CaptureCandidate[] {
+  const seen = new Set<string>();
+  const deduped: CaptureCandidate[] = [];
+  for (const candidate of candidates) {
+    if (seen.has(candidate.url)) continue;
+    seen.add(candidate.url);
+    deduped.push(candidate);
+  }
+  return deduped;
+}