tgo-wiki 0.1.0

package/packages/core/src/documents/command-document-parser.ts

@@ -0,0 +1,206 @@
+import { spawn, type ChildProcessWithoutNullStreams } from "node:child_process";
+import { WikiError } from "../errors.js";
+import type { DocumentParseInput, DocumentParseOutput, DocumentParser } from "./document-parser-registry.js";
+
+const defaultTimeoutMs = 30_000;
+const outputSnippetLimit = 4_096;
+
+export type CommandDocumentParserOptions = {
+  name: string;
+  command: string;
+  supportedMimeTypes: string[];
+  version?: string;
+  timeoutMs?: number;
+};
+
+export class CommandDocumentParser implements DocumentParser {
+  readonly name: string;
+  readonly version: string;
+  readonly supportedMimeTypes: readonly string[];
+  private readonly command: string;
+  private readonly executable: string;
+  private readonly args: string[];
+  private readonly timeoutMs: number;
+
+  constructor(options: CommandDocumentParserOptions) {
+    this.name = options.name;
+    this.command = options.command;
+    this.version = options.version ?? "command";
+    this.supportedMimeTypes = options.supportedMimeTypes;
+    this.timeoutMs = options.timeoutMs ?? defaultTimeoutMs;
+    const [executable, ...args] = splitCommandLine(options.command);
+    if (!executable) {
+      throw new WikiError("parser_not_supported", `Invalid parser command: ${options.name}`, { parser: options.name });
+    }
+    this.executable = executable;
+    this.args = args;
+  }
+
+  async parse(input: DocumentParseInput): Promise<DocumentParseOutput> {
+    const result = await runParserCommand({
+      executable: this.executable,
+      args: this.args,
+      input,
+      timeoutMs: this.timeoutMs
+    });
+    const stdout = stripUtf8Bom(result.stdout.toString("utf8"));
+    const stderr = result.stderr.toString("utf8");
+
+    if (stdout.length === 0) {
+      throw new WikiError("parse_failed", `Command parser produced empty stdout: ${this.name}`, { parser: this.name });
+    }
+
+    const markdown = ensureFinalNewline(stdout);
+
+    return {
+      markdown,
+      assets: [],
+      warnings: stderr.length > 0 ? [truncate(stderr.trimEnd())] : [],
+      metadata: {
+        command: this.command,
+        stdoutBytes: Buffer.byteLength(markdown),
+        stderrBytes: result.stderr.length
+      }
+    };
+  }
+}
+
+export function splitCommandLine(command: string): string[] {
+  const tokens: string[] = [];
+  let current = "";
+  let quote: "'" | '"' | undefined;
+
+  for (let i = 0; i < command.length; i += 1) {
+    const char = command[i];
+    if (quote) {
+      if (char === quote) {
+        quote = undefined;
+      } else {
+        current += char;
+      }
+      continue;
+    }
+
+    if (char === "'" || char === '"') {
+      quote = char;
+      continue;
+    }
+
+    if (/\s/.test(char)) {
+      if (current.length > 0) {
+        tokens.push(current);
+        current = "";
+      }
+      continue;
+    }
+
+    current += char;
+  }
+
+  if (quote) {
+    throw new WikiError("parser_not_supported", "Invalid parser command: unterminated quote");
+  }
+
+  if (current.length > 0) {
+    tokens.push(current);
+  }
+
+  return tokens;
+}
+
+type RunParserCommandInput = {
+  executable: string;
+  args: string[];
+  input: DocumentParseInput;
+  timeoutMs: number;
+};
+
+type RunParserCommandOutput = {
+  stdout: Buffer;
+  stderr: Buffer;
+};
+
+function runParserCommand(options: RunParserCommandInput): Promise<RunParserCommandOutput> {
+  return new Promise((resolve, reject) => {
+    let child: ChildProcessWithoutNullStreams;
+    try {
+      child = spawn(options.executable, options.args, {
+        shell: false,
+        stdio: ["pipe", "pipe", "pipe"],
+        env: {
+          ...process.env,
+          TGO_DOCUMENT_ID: options.input.documentId,
+          TGO_FILE_NAME: options.input.fileName,
+          TGO_MIME_TYPE: options.input.mimeType,
+          TGO_PARSED_AT: options.input.parsedAt
+        }
+      }) as ChildProcessWithoutNullStreams;
+    } catch (error) {
+      reject(new WikiError("parse_failed", error instanceof Error ? error.message : String(error)));
+      return;
+    }
+    const stdout: Buffer[] = [];
+    const stderr: Buffer[] = [];
+    let timedOut = false;
+    let settled = false;
+
+    const timeout = setTimeout(() => {
+      timedOut = true;
+      child.kill();
+    }, options.timeoutMs);
+
+    const rejectOnce = (error: WikiError) => {
+      if (settled) {
+        return;
+      }
+
+      settled = true;
+      clearTimeout(timeout);
+      reject(error);
+    };
+
+    child.stdout.on("data", chunk => stdout.push(Buffer.from(chunk)));
+    child.stderr.on("data", chunk => stderr.push(Buffer.from(chunk)));
+    child.on("error", error => {
+      rejectOnce(new WikiError("parse_failed", error.message));
+    });
+    child.on("close", code => {
+      const stderrBuffer = Buffer.concat(stderr);
+      if (timedOut) {
+        rejectOnce(new WikiError("parse_failed", "Command parser timed out", { stderr: truncate(stderrBuffer.toString("utf8")) }));
+        return;
+      }
+      if (code !== 0) {
+        rejectOnce(
+          new WikiError("parse_failed", `Command parser exited with code ${code}`, {
+            exitCode: code,
+            stderr: truncate(stderrBuffer.toString("utf8"))
+          })
+        );
+        return;
+      }
+
+      if (settled) {
+        return;
+      }
+
+      settled = true;
+      clearTimeout(timeout);
+      resolve({ stdout: Buffer.concat(stdout), stderr: stderrBuffer });
+    });
+
+    child.stdin.end(options.input.content);
+  });
+}
+
+function ensureFinalNewline(value: string): string {
+  return value.endsWith("\n") ? value : `${value}\n`;
+}
+
+function stripUtf8Bom(value: string): string {
+  return value.startsWith("\uFEFF") ? value.slice(1) : value;
+}
+
+function truncate(value: string): string {
+  return value.length > outputSnippetLimit ? `${value.slice(0, outputSnippetLimit)}...` : value;
+}

package/packages/core/src/documents/document-id.ts

@@ -0,0 +1,26 @@
+import { randomBytes } from "node:crypto";
+import path from "node:path";
+import { WikiError } from "../errors.js";
+
+const documentIdPattern = /^[a-zA-Z0-9._-]+$/;
+
+export function generateDocumentId(now = new Date()): string {
+  const date = now.toISOString().slice(0, 10).replaceAll("-", "");
+  const suffix = randomBytes(6).toString("hex");
+  return `doc_${date}_${suffix}`;
+}
+
+export function assertValidDocumentId(documentId: string): void {
+  if (
+    documentId.length === 0 ||
+    !documentIdPattern.test(documentId) ||
+    documentId.includes("/") ||
+    documentId.includes("\\") ||
+    documentId === "." ||
+    documentId === ".." ||
+    path.isAbsolute(documentId) ||
+    path.win32.isAbsolute(documentId)
+  ) {
+    throw new WikiError("invalid_path", `Invalid document id: ${documentId}`);
+  }
+}

package/packages/core/src/documents/document-parser-registry.ts

@@ -0,0 +1,126 @@
+import type { DocumentParserConfig } from "../config/defaults.js";
+import { WikiError } from "../errors.js";
+import { CommandDocumentParser } from "./command-document-parser.js";
+import { PdfTextParser } from "./pdf-text-parser.js";
+import { TextMarkdownParser } from "./text-markdown-parser.js";
+
+export type DocumentParserConfigShape = {
+  defaultParser: string;
+  parsers: Record<string, DocumentParserConfig>;
+};
+
+export type DocumentParserAsset = {
+  path: string;
+  content: Buffer | Uint8Array | string;
+};
+
+export type DocumentParseInput = {
+  documentId: string;
+  fileName: string;
+  mimeType: string;
+  content: Buffer;
+  parsedAt: string;
+};
+
+export type DocumentParseOutput = {
+  markdown: string;
+  assets: DocumentParserAsset[];
+  warnings: string[];
+  metadata: Record<string, unknown>;
+};
+
+export type DocumentParser = {
+  name: string;
+  version: string;
+  supportedMimeTypes: readonly string[];
+  parse(input: DocumentParseInput): Promise<DocumentParseOutput>;
+};
+
+export class DocumentParserRegistry {
+  private readonly parsers: Map<string, DocumentParser>;
+
+  constructor(
+    private readonly config: DocumentParserConfigShape,
+    parsers: Iterable<DocumentParser> = [new PdfTextParser(), new TextMarkdownParser()]
+  ) {
+    this.parsers = new Map(Array.from(parsers, parser => [parser.name, parser]));
+
+    for (const [parserName, parserConfig] of Object.entries(config.parsers)) {
+      if (this.parsers.has(parserName) || parserConfig.enabled !== true || !parserConfig.command) {
+        continue;
+      }
+
+      if (!parserConfig.supportedMimeTypes || parserConfig.supportedMimeTypes.length === 0) {
+        continue;
+      }
+
+      this.parsers.set(
+        parserName,
+        new CommandDocumentParser({
+          name: parserName,
+          command: parserConfig.command,
+          supportedMimeTypes: parserConfig.supportedMimeTypes,
+          version: parserConfig.version,
+          timeoutMs: parserConfig.timeoutMs
+        })
+      );
+    }
+  }
+
+  resolve(requestedParser: "auto" | string, mimeType: string): DocumentParser {
+    const parserName = requestedParser === "auto" ? this.resolveAutoParserName(mimeType) : requestedParser;
+    const configured = this.config.parsers[parserName];
+
+    if (!configured) {
+      throw new WikiError("parser_not_supported", `Document parser is not supported: ${parserName}`, {
+        parser: parserName
+      });
+    }
+
+    if (!configured.enabled) {
+      throw new WikiError("parser_not_supported", `Document parser is disabled: ${parserName}`, {
+        parser: parserName
+      });
+    }
+
+    const parser = this.parsers.get(parserName);
+    if (!parser) {
+      throw new WikiError("parser_not_supported", `Document parser is not supported: ${parserName}`, {
+        parser: parserName
+      });
+    }
+
+    if (!parser.supportedMimeTypes.includes(mimeType)) {
+      throw new WikiError("parser_not_supported", `Document parser does not support MIME type: ${mimeType}`, {
+        parser: parserName,
+        mimeType
+      });
+    }
+
+    return parser;
+  }
+
+  private resolveAutoParserName(mimeType: string): string {
+    const defaultParser = this.parserIfEnabled(this.config.defaultParser);
+    if (defaultParser?.supportedMimeTypes.includes(mimeType)) {
+      return defaultParser.name;
+    }
+
+    for (const parserName of Object.keys(this.config.parsers)) {
+      const parser = this.parserIfEnabled(parserName);
+      if (parser?.supportedMimeTypes.includes(mimeType)) {
+        return parser.name;
+      }
+    }
+
+    return this.config.defaultParser;
+  }
+
+  private parserIfEnabled(parserName: string): DocumentParser | undefined {
+    if (this.config.parsers[parserName]?.enabled !== true) {
+      return undefined;
+    }
+
+    return this.parsers.get(parserName);
+  }
+}