tgo-wiki 0.1.0

package/packages/core/src/vfs/vfs-service.ts

@@ -0,0 +1,498 @@
+import { lstat, readdir, readFile, realpath } from "node:fs/promises";
+import path from "node:path";
+import { resolveWorkspacePaths, type WorkspacePaths } from "../config/workspace-resolver.js";
+import { WikiError } from "../errors.js";
+import { err, ok, type Result } from "../result.js";
+import { parseVfsCommand, type VfsCommand } from "./vfs-command-parser.js";
+
+export type VfsExecInput = {
+  command: string;
+  ref?: "stable" | string;
+};
+
+export type VfsExecResult = {
+  stdout: string;
+  stderr: string;
+  exit_code: number;
+  cwd: "wiki";
+  ref: "stable";
+};
+
+export class VfsService {
+  private readonly paths: WorkspacePaths;
+
+  constructor(workspaceRoot: string) {
+    this.paths = resolveWorkspacePaths(workspaceRoot);
+  }
+
+  async exec(input: VfsExecInput): Promise<Result<VfsExecResult>> {
+    if ((input.ref ?? "stable") !== "stable") {
+      return err(new WikiError("unsupported_channel", `Unsupported VFS ref: ${input.ref}`));
+    }
+
+    try {
+      const command = parseVfsCommand(input.command);
+      const stdout = await this.run(command);
+
+      return ok({
+        stdout,
+        stderr: "",
+        exit_code: 0,
+        cwd: "wiki",
+        ref: "stable"
+      });
+    } catch (error) {
+      if (error instanceof WikiError) {
+        return err(error);
+      }
+
+      return err(new WikiError("invalid_path", error instanceof Error ? error.message : String(error)));
+    }
+  }
+
+  private async run(command: VfsCommand): Promise<string> {
+    switch (command.name) {
+      case "pwd":
+        return "wiki\n";
+      case "ls":
+        return this.ls(command.args);
+      case "find":
+        return this.find(command.args);
+      case "tree":
+        return this.tree(command.args);
+      case "cat":
+        return this.cat(command.args);
+      case "head":
+        return this.head(command.args);
+      case "tail":
+        return this.tail(command.args);
+      case "wc":
+        return this.wc(command.args);
+      case "grep":
+        return this.grep(command.args);
+    }
+  }
+
+  private async ls(args: string[]): Promise<string> {
+    const parsed = parseLsArgs(args);
+    const target = await this.resolveDirectory(parsed.path);
+    const entries = await readdir(target, { withFileTypes: true });
+
+    return entries
+      .map(entry => `${entry.name}${entry.isDirectory() ? "/" : ""}`)
+      .sort()
+      .join("\n")
+      .concat(entries.length > 0 ? "\n" : "");
+  }
+
+  private async find(args: string[]): Promise<string> {
+    const parsed = parseFindArgs(args);
+    const root = await this.resolveDirectory(parsed.path);
+    const files = await this.collectMarkdownFiles(root, parsed.extensions, parsed.maxDepth);
+
+    return formatLines(files);
+  }
+
+  private async tree(args: string[]): Promise<string> {
+    const parsed = parseTreeArgs(args);
+    const target = await this.resolveDirectory(parsed.path);
+    const files = await this.collectMarkdownFiles(target);
+
+    return formatLines(files.map(file => file.replace(/^wiki\//, "")));
+  }
+
+  private async cat(args: string[]): Promise<string> {
+    const file = await this.readCommandFile(parseSingleFilePath(args, "cat"));
+    return file.content;
+  }
+
+  private async head(args: string[]): Promise<string> {
+    const { count, paths } = parseLineCount(args);
+    const file = await this.readCommandFile(singlePathOrRequired(paths, "head"));
+    return firstLines(file.content, count);
+  }
+
+  private async tail(args: string[]): Promise<string> {
+    const { count, paths } = parseLineCount(args);
+    const file = await this.readCommandFile(singlePathOrRequired(paths, "tail"));
+    return lastLines(file.content, count);
+  }
+
+  private async wc(args: string[]): Promise<string> {
+    const file = await this.readCommandFile(parseSingleFilePath(args, "wc"));
+    const lines = file.content.length === 0 ? 0 : file.content.split("\n").length - (file.content.endsWith("\n") ? 1 : 0);
+    const words = file.content.trim().length === 0 ? 0 : file.content.trim().split(/\s+/).length;
+    const bytes = Buffer.byteLength(file.content, "utf8");
+
+    return `${lines} ${words} ${bytes} ${file.vfsPath}\n`;
+  }
+
+  private async grep(args: string[]): Promise<string> {
+    const parsed = parseGrepArgs(args);
+
+    if (parsed.paths.length === 0) {
+      parsed.paths.push("wiki");
+    }
+
+    const matches: string[] = [];
+    const files = await this.expandGrepFiles(parsed.paths, parsed.recursive);
+
+    for (const filePath of files) {
+      const file = await this.readCommandFile(filePath);
+      const lines = file.content.split("\n");
+
+      for (const [index, line] of lines.entries()) {
+        if (line.includes(parsed.pattern)) {
+          const prefix = files.length > 1 || parsed.recursive ? `${file.vfsPath}:` : "";
+          const lineNumber = parsed.lineNumbers ? `${index + 1}:` : "";
+          matches.push(`${prefix}${lineNumber}${line}`);
+        }
+      }
+    }
+
+    return formatLines(matches);
+  }
+
+  private async readCommandFile(input: string | undefined): Promise<{ content: string; vfsPath: string }> {
+    const vfsPath = normalizeVfsPath(input);
+    const filePath = await this.resolveFile(vfsPath);
+    const content = await readFile(filePath, "utf8");
+
+    return { content, vfsPath };
+  }
+
+  private async resolveFile(input: string): Promise<string> {
+    const filePath = this.resolvePath(input);
+    await ensureReadableTargetInsideWikiRoot(filePath, this.wikiRootPath());
+    return filePath;
+  }
+
+  private async resolveDirectory(input: string): Promise<string> {
+    const directoryPath = this.resolvePath(input);
+    await ensureReadableTargetInsideWikiRoot(directoryPath, this.wikiRootPath());
+    const stat = await lstat(directoryPath);
+
+    if (!stat.isDirectory()) {
+      throw new WikiError("invalid_path", `Path must be a directory: ${input}`);
+    }
+
+    return directoryPath;
+  }
+
+  private resolvePath(input: string): string {
+    const vfsPath = normalizeVfsPath(input);
+    return path.resolve(this.paths.stableWorktreePath, vfsPath);
+  }
+
+  private wikiRootPath(): string {
+    return path.join(this.paths.stableWorktreePath, this.paths.wikiRootName);
+  }
+
+  private async expandGrepFiles(inputs: string[], recursive: boolean): Promise<string[]> {
+    const files: string[] = [];
+
+    for (const input of inputs) {
+      const absolute = this.resolvePath(input);
+      await ensureReadableTargetInsideWikiRoot(absolute, this.wikiRootPath());
+      const stat = await lstat(absolute);
+
+      if (stat.isDirectory()) {
+        if (!recursive) {
+          throw new WikiError("vfs_command_denied", `grep requires -R for directories: ${input}`);
+        }
+
+        files.push(...(await this.collectMarkdownFiles(absolute)));
+        continue;
+      }
+
+      if (stat.isFile()) {
+        files.push(toVfsPath(absolute, this.paths.stableWorktreePath));
+        continue;
+      }
+
+      throw new WikiError("invalid_path", `Path must be a file or directory: ${input}`);
+    }
+
+    return files.sort();
+  }
+
+  private async collectMarkdownFiles(
+    root: string,
+    extensions = [".md"],
+    maxDepth = Number.POSITIVE_INFINITY,
+    depth = 0
+  ): Promise<string[]> {
+    const entries = await readdir(root, { withFileTypes: true });
+    const files = await Promise.all(
+      entries.map(async entry => {
+        const absolute = path.join(root, entry.name);
+        await ensureReadableTargetInsideWikiRoot(absolute, this.wikiRootPath());
+
+        if (entry.isDirectory()) {
+          return depth + 1 < maxDepth ? this.collectMarkdownFiles(absolute, extensions, maxDepth, depth + 1) : [];
+        }
+
+        if (entry.isFile() && extensions.some(extension => entry.name.endsWith(extension))) {
+          return [toVfsPath(absolute, this.paths.stableWorktreePath)];
+        }
+
+        return [];
+      })
+    );
+
+    return files.flat().sort();
+  }
+}
+
+function normalizeVfsPath(input: string | undefined): string {
+  if (!input) {
+    throw new WikiError("invalid_path", "Path is required");
+  }
+
+  const unixInput = input.replaceAll("\\", "/");
+
+  if (path.posix.isAbsolute(unixInput) || /^[A-Za-z]:\//.test(unixInput)) {
+    throw new WikiError("invalid_path", `Path must be relative: ${input}`);
+  }
+
+  if (unixInput.split("/").some(segment => segment === "..")) {
+    throw new WikiError("invalid_path", `Path escapes wiki root: ${input}`);
+  }
+
+  const normalized = path.posix.normalize(unixInput);
+  const vfsPath = normalized === "." || normalized === "wiki" || normalized.startsWith("wiki/")
+    ? normalized === "." ? "wiki" : normalized
+    : `wiki/${normalized}`;
+
+  if (vfsPath !== "wiki" && !vfsPath.startsWith("wiki/")) {
+    throw new WikiError("invalid_path", `Path must be inside wiki/: ${input}`);
+  }
+
+  return vfsPath;
+}
+
+function parseLsArgs(args: string[]): { path: string } {
+  const paths: string[] = [];
+
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index];
+
+    if (arg === "--color") {
+      const value = args[index + 1];
+
+      if (value !== "never") {
+        throw new WikiError("vfs_command_denied", "--color requires never");
+      }
+
+      index += 1;
+      continue;
+    }
+
+    if (arg.startsWith("-")) {
+      throw new WikiError("vfs_command_denied", `Unsupported ls option: ${arg}`);
+    }
+
+    paths.push(arg);
+  }
+
+  return { path: singlePathOrDefault(paths, "ls") };
+}
+
+function parseFindArgs(args: string[]): { path: string; extensions: string[]; maxDepth: number } {
+  const paths: string[] = [];
+  let extensions = [".md"];
+  let maxDepth = Number.POSITIVE_INFINITY;
+
+  for (let index = 0; index < args.length; index += 1) {
+    const arg = args[index];
+
+    if (arg === "-name") {
+      const value = args[index + 1];
+
+      if (!value || value.startsWith("-")) {
+        throw new WikiError("vfs_command_denied", "-name requires a pattern");
+      }
+
+      if (!["*.md", "*.markdown"].includes(value)) {
+        throw new WikiError("vfs_command_denied", `Unsupported find -name pattern: ${value}`);
+      }
+
+      extensions = [value.slice(1)];
+      index += 1;
+      continue;
+    }
+
+    if (arg === "-maxdepth") {
+      const value = args[index + 1];
+      const parsed = Number.parseInt(value ?? "", 10);
+
+      if (!Number.isInteger(parsed) || parsed < 0 || String(parsed) !== value) {
+        throw new WikiError("vfs_command_denied", "-maxdepth requires a non-negative integer");
+      }
+
+      maxDepth = parsed;
+      index += 1;
+      continue;
+    }
+
+    if (arg.startsWith("-")) {
+      throw new WikiError("vfs_command_denied", `Unsupported find option: ${arg}`);
+    }
+
+    paths.push(arg);
+  }
+
+  return { path: singlePathOrDefault(paths, "find"), extensions, maxDepth };
+}
+
+function parseTreeArgs(args: string[]): { path: string } {
+  const paths: string[] = [];
+
+  for (const arg of args) {
+    if (arg.startsWith("-")) {
+      throw new WikiError("vfs_command_denied", `Unsupported tree option: ${arg}`);
+    }
+
+    paths.push(arg);
+  }
+
+  return { path: singlePathOrDefault(paths, "tree") };
+}
+
+function parseGrepArgs(args: string[]): { pattern: string; paths: string[]; lineNumbers: boolean; recursive: boolean } {
+  let lineNumbers = false;
+  let recursive = false;
+  let index = 0;
+
+  while (index < args.length) {
+    const arg = args[index];
+
+    if (arg === "-n") {
+      lineNumbers = true;
+      index += 1;
+      continue;
+    }
+
+    if (arg === "-R" || arg === "-r") {
+      recursive = true;
+      index += 1;
+      continue;
+    }
+
+    if (arg.startsWith("-")) {
+      throw new WikiError("vfs_command_denied", `Unsupported grep option: ${arg}`);
+    }
+
+    break;
+  }
+
+  const pattern = args[index];
+
+  if (!pattern) {
+    throw new WikiError("vfs_command_denied", "grep requires a pattern");
+  }
+
+  const paths = args.slice(index + 1);
+
+  for (const pathArg of paths) {
+    if (pathArg.startsWith("-")) {
+      throw new WikiError("vfs_command_denied", `Unsupported grep option: ${pathArg}`);
+    }
+  }
+
+  return {
+    pattern,
+    paths,
+    lineNumbers,
+    recursive
+  };
+}
+
+function singlePathOrDefault(paths: string[], command: string): string {
+  if (paths.length > 1) {
+    throw new WikiError("vfs_command_denied", `${command} accepts at most one path`);
+  }
+
+  return paths[0] ?? "wiki";
+}
+
+function parseSingleFilePath(args: string[], command: string): string {
+  for (const arg of args) {
+    if (arg.startsWith("-")) {
+      throw new WikiError("vfs_command_denied", `Unsupported ${command} option: ${arg}`);
+    }
+  }
+
+  return singlePathOrRequired(args, command);
+}
+
+function singlePathOrRequired(paths: string[], command: string): string {
+  for (const pathArg of paths) {
+    if (pathArg.startsWith("-")) {
+      throw new WikiError("vfs_command_denied", `Unsupported ${command} option: ${pathArg}`);
+    }
+  }
+
+  if (paths.length !== 1) {
+    throw new WikiError("vfs_command_denied", `${command} requires exactly one path`);
+  }
+
+  return paths[0];
+}
+
+async function ensureReadableTargetInsideWikiRoot(filePath: string, wikiRootPath: string): Promise<void> {
+  const stat = await lstat(filePath);
+
+  if (stat.isSymbolicLink()) {
+    throw new WikiError("invalid_path", `Path is a symlink: ${filePath}`);
+  }
+
+  const wikiRoot = await realpath(wikiRootPath);
+  const target = await realpath(filePath);
+
+  if (!target.startsWith(`${wikiRoot}${path.sep}`) && target !== wikiRoot) {
+    throw new WikiError("invalid_path", `Path escapes wiki root: ${filePath}`);
+  }
+}
+
+function parseLineCount(args: string[]): { count: number; paths: string[] } {
+  if (args[0] === "-n") {
+    const count = Number.parseInt(args[1] ?? "", 10);
+
+    if (!Number.isInteger(count) || count < 0) {
+      throw new WikiError("vfs_command_denied", "Line count must be a non-negative integer");
+    }
+
+    return { count, paths: args.slice(2) };
+  }
+
+  if (args.some(arg => arg.startsWith("-"))) {
+    const option = args.find(arg => arg.startsWith("-"));
+    throw new WikiError("vfs_command_denied", `Unsupported line option: ${option}`);
+  }
+
+  return { count: 10, paths: args };
+}
+
+function firstLines(content: string, count: number): string {
+  return ensureTrailingNewline(content.split("\n").slice(0, count).join("\n"));
+}
+
+function lastLines(content: string, count: number): string {
+  const lines = content.split("\n");
+  const trimmed = content.endsWith("\n") ? lines.slice(0, -1) : lines;
+
+  return ensureTrailingNewline(trimmed.slice(Math.max(0, trimmed.length - count)).join("\n"));
+}
+
+function ensureTrailingNewline(content: string): string {
+  return content.length > 0 && !content.endsWith("\n") ? `${content}\n` : content;
+}
+
+function formatLines(lines: string[]): string {
+  return lines.length > 0 ? `${lines.join("\n")}\n` : "";
+}
+
+function toVfsPath(filePath: string, stableWorktreePath: string): string {
+  return path.relative(stableWorktreePath, filePath).split(path.sep).join("/");
+}

package/packages/core/src/web/html-to-markdown.ts

@@ -0,0 +1,144 @@
+export type HtmlToRawMarkdownInput = {
+  html: string;
+  requestedUrl: string;
+  finalUrl: string;
+  fetchedAt: string;
+  fetcherName: string;
+};
+
+export type HtmlToRawMarkdownResult = {
+  title?: string;
+  markdown: string;
+};
+
+type BlockTag = "h1" | "h2" | "h3" | "p" | "li";
+
+type MarkdownBlock = {
+  text: string;
+  isListItem: boolean;
+};
+
+export function htmlToRawMarkdown(input: HtmlToRawMarkdownInput): HtmlToRawMarkdownResult {
+  const cleanedHtml = removeIgnoredElements(input.html);
+  const title = extractFirstTagText(cleanedHtml, "title");
+  const firstH1 = extractFirstTagText(cleanedHtml, "h1");
+  const pageTitle = title ?? firstH1;
+  const headerText = pageTitle ?? input.finalUrl;
+  const bodyMarkdown = renderBodyMarkdown(cleanedHtml, headerText);
+
+  const markdown = normalizeMarkdown(`# ${headerText}
+
+> Source web page: ${input.requestedUrl}
+> Final URL: ${input.finalUrl}
+> Fetched at: ${input.fetchedAt}
+> Fetcher: ${input.fetcherName}
+
+${bodyMarkdown}`);
+
+  return {
+    title: pageTitle,
+    markdown
+  };
+}
+
+function removeIgnoredElements(html: string): string {
+  return html.replace(/<(script|style|noscript)\b[^>]*>[\s\S]*?<\/\1>/gi, "");
+}
+
+function extractFirstTagText(html: string, tag: "title" | "h1"): string | undefined {
+  const match = new RegExp(`<${tag}\\b[^>]*>([\\s\\S]*?)<\\/${tag}>`, "i").exec(html);
+  if (!match) {
+    return undefined;
+  }
+
+  const text = inlineText(match[1]);
+  return text.length > 0 ? text : undefined;
+}
+
+function renderBodyMarkdown(html: string, headerText: string): string {
+  const bodyHtml = html
+    .replace(/<!doctype\b[^>]*>/gi, "")
+    .replace(/<head\b[^>]*>[\s\S]*?<\/head>/gi, "")
+    .replace(/<title\b[^>]*>[\s\S]*?<\/title>/gi, "");
+  const blocks: MarkdownBlock[] = [];
+  const blockPattern = /<(h1|h2|h3|p|li)\b[^>]*>([\s\S]*?)<\/\1>/gi;
+  let lastIndex = 0;
+
+  for (const match of bodyHtml.matchAll(blockPattern)) {
+    addPlainTextBlock(blocks, bodyHtml.slice(lastIndex, match.index));
+
+    const tag = match[1].toLowerCase() as BlockTag;
+    const text = inlineText(match[2]);
+    lastIndex = match.index + match[0].length;
+
+    if (text.length === 0 || (tag === "h1" && text === headerText)) {
+      continue;
+    }
+
+    if (tag === "h1") {
+      blocks.push({ text: `# ${text}`, isListItem: false });
+    } else if (tag === "h2") {
+      blocks.push({ text: `## ${text}`, isListItem: false });
+    } else if (tag === "h3") {
+      blocks.push({ text: `### ${text}`, isListItem: false });
+    } else if (tag === "li") {
+      blocks.push({ text: `- ${text}`, isListItem: true });
+    } else {
+      blocks.push({ text, isListItem: false });
+    }
+  }
+
+  addPlainTextBlock(blocks, bodyHtml.slice(lastIndex));
+
+  return joinMarkdownBlocks(blocks);
+}
+
+function addPlainTextBlock(blocks: MarkdownBlock[], html: string): void {
+  const text = inlineText(html);
+  if (text.length > 0) {
+    blocks.push({ text, isListItem: false });
+  }
+}
+
+function joinMarkdownBlocks(blocks: MarkdownBlock[]): string {
+  return blocks.reduce((markdown, block, index) => {
+    if (index === 0) {
+      return block.text;
+    }
+
+    const previous = blocks[index - 1];
+    const separator = previous.isListItem && block.isListItem ? "\n" : "\n\n";
+    return `${markdown}${separator}${block.text}`;
+  }, "");
+}
+
+function inlineText(html: string): string {
+  const withLinks = html.replace(/<a\b([^>]*)>([\s\S]*?)<\/a>/gi, (_match, attrs: string, content: string) => {
+    const text = inlineText(content);
+    const href = extractHref(attrs);
+    return href ? `[${text}](${decodeHtmlEntities(href)})` : text;
+  });
+
+  return decodeHtmlEntities(withLinks.replace(/<[^>]+>/g, " "))
+    .replace(/\s+/g, " ")
+    .trim();
+}
+
+function extractHref(attrs: string): string | undefined {
+  const match = /\bhref\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s"'>]+))/i.exec(attrs);
+  return match?.[1] ?? match?.[2] ?? match?.[3];
+}
+
+function decodeHtmlEntities(text: string): string {
+  return text
+    .replace(/&nbsp;/g, " ")
+    .replace(/&amp;/g, "&")
+    .replace(/&lt;/g, "<")
+    .replace(/&gt;/g, ">")
+    .replace(/&quot;/g, "\"")
+    .replace(/&#39;/g, "'");
+}
+
+function normalizeMarkdown(markdown: string): string {
+  return `${markdown.replace(/[ \t]+\n/g, "\n").replace(/\n{3,}/g, "\n\n").trim()}\n`;
+}