tgo-wiki 0.1.0

package/packages/core/src/documents/document-service.ts

@@ -0,0 +1,656 @@
+import { createHash, randomBytes } from "node:crypto";
+import { lstat, mkdir, readFile, realpath, rm, writeFile } from "node:fs/promises";
+import path from "node:path";
+import { loadConfig } from "../config/config-loader.js";
+import { resolveWorkspacePaths, type WorkspacePaths } from "../config/workspace-resolver.js";
+import { WikiError, toWikiError } from "../errors.js";
+import { err, ok, type Result } from "../result.js";
+import { SessionStore } from "../session/session-store.js";
+import type { SessionMetadata } from "../session/session-types.js";
+import { sourceMetadataPath } from "../sources/source-paths.js";
+import type { SourceMetadata } from "../sources/source-types.js";
+import { parseSourceMetadata } from "../sources/source-validator.js";
+import { assertNoExistingSourceForCreate, publishSource } from "../sources/source-writer.js";
+import { assertValidDocumentId, generateDocumentId } from "./document-id.js";
+import { DocumentParserRegistry } from "./document-parser-registry.js";
+import { DocumentStore } from "./document-store.js";
+import type {
+  DocumentIdGenerator,
+  DocumentSourceType,
+  DocumentUploadInput,
+  DocumentUploadResult,
+  PendingDocumentMetadata
+} from "./document-types.js";
+
+export type DocumentParseInput = {
+  sessionId: string;
+  documentId: string;
+  parser?: "auto" | string;
+  reparse?: boolean;
+};
+
+export type DocumentParseResult = {
+  document_id: string;
+  version: number;
+  raw_markdown_path: string;
+  metadata_path: string;
+  status: "parsed";
+  warnings: string[];
+};
+
+export type DocumentServiceOptions = {
+  parserRegistry?: DocumentParserRegistry;
+  clock?: () => Date;
+};
+
+export class DocumentService {
+  private readonly paths: WorkspacePaths;
+  private readonly sessions: SessionStore;
+  private readonly documents: DocumentStore;
+  private readonly parserRegistry?: DocumentParserRegistry;
+  private readonly clock: () => Date;
+
+  constructor(
+    workspaceRoot: string,
+    private readonly idGenerator: DocumentIdGenerator = generateDocumentId,
+    options: DocumentServiceOptions = {}
+  ) {
+    this.paths = resolveWorkspacePaths(workspaceRoot);
+    this.sessions = new SessionStore(this.paths);
+    this.documents = new DocumentStore(this.paths);
+    this.parserRegistry = options.parserRegistry;
+    this.clock = options.clock ?? (() => new Date());
+  }
+
+  async upload(input: DocumentUploadInput): Promise<Result<DocumentUploadResult>> {
+    try {
+      const session = await this.sessions.read(input.sessionId);
+      if (session.status !== "open") {
+        throw new WikiError("validation_failed", `Session is not open: ${input.sessionId}`, {
+          status: session.status
+        });
+      }
+
+      const documentId = this.idGenerator();
+      assertValidDocumentId(documentId);
+      const sourceType = sourceTypeForMimeType(input.mimeType);
+      const content = decodeBase64(input.contentBase64);
+      const blobSha256 = createHash("sha256").update(content).digest("hex");
+
+      await this.documents.writeBlobIfAbsent(blobSha256, content);
+
+      const now = new Date().toISOString();
+      const metadata: PendingDocumentMetadata = {
+        documentId,
+        sessionId: session.sessionId,
+        sourceType,
+        originalFileName: input.fileName,
+        mimeType: input.mimeType,
+        blobSha256,
+        createdAt: now,
+        updatedAt: now,
+        createdBy: input.createdBy,
+        status: "uploaded"
+      };
+      await this.documents.writePending(metadata);
+
+      return ok({
+        document_id: documentId,
+        blob_sha256: blobSha256,
+        size: content.byteLength,
+        status: "uploaded"
+      });
+    } catch (error) {
+      return err(toWikiError(error, "validation_failed"));
+    }
+  }
+
+  async parse(input: DocumentParseInput): Promise<Result<DocumentParseResult>> {
+    let lock: ParseLock | undefined;
+
+    try {
+      assertValidDocumentId(input.documentId);
+      const session = await this.sessions.read(input.sessionId);
+      if (session.status !== "open") {
+        throw new WikiError("validation_failed", `Session is not open: ${input.sessionId}`, {
+          status: session.status
+        });
+      }
+
+      const worktreeRoot = await this.sessionWorktreeRoot(session);
+      lock = await acquireParseLock(this.paths, input.sessionId, input.documentId);
+      const existing = await this.readExistingSourceMetadataIfExists(worktreeRoot, input.documentId);
+      if (existing && input.reparse !== true) {
+        throw new WikiError("source_exists", `Source already exists: ${input.documentId}`);
+      }
+
+      let document: PendingDocumentMetadata | SourceMetadata | undefined = existing;
+      if (!document) {
+        document = await this.readPendingIfExists(input.documentId, input.sessionId);
+      }
+
+      if (!document) {
+        throw new WikiError("document_not_found", `Document metadata not found: ${input.documentId}`);
+      }
+
+      if (document.sourceType === "web") {
+        throw new WikiError("validation_failed", `Cannot parse web source as document: ${input.documentId}`);
+      }
+
+      if (input.reparse !== true) {
+        await assertNoExistingSourceForCreate(worktreeRoot, input.documentId);
+      }
+
+      const blob = await this.readBlob(document.blobSha256);
+      const registry = await this.resolveParserRegistry();
+      const requestedParser = input.parser ?? "auto";
+      const parser = registry.resolve(requestedParser, document.mimeType);
+      const parsedAt = this.clock().toISOString();
+      const parsed = await parser.parse({
+        documentId: input.documentId,
+        fileName: document.originalFileName,
+        mimeType: document.mimeType,
+        content: blob,
+        parsedAt
+      });
+
+      const now = parsedAt;
+      const relativeRawMarkdownPath = sourceRelativePath(input.documentId, "raw.md");
+      const relativeMetadataPath = sourceRelativePath(input.documentId, "metadata.json");
+      const metadata = {
+        documentId: input.documentId,
+        version: existing && input.reparse ? existing.version + 1 : 1,
+        sourceType: document.sourceType,
+        originalFileName: document.originalFileName,
+        mimeType: document.mimeType,
+        blobSha256: document.blobSha256,
+        rawMarkdownPath: relativeRawMarkdownPath,
+        parser: {
+          name: parser.name,
+          version: parser.version
+        },
+        parserMetadata: parsed.metadata,
+        createdAt: existing?.createdAt ?? document.createdAt,
+        updatedAt: now,
+        createdBy: document.createdBy,
+        status: "parsed"
+      };
+
+      await publishSource({
+        worktreeRoot,
+        sourceId: input.documentId,
+        markdown: parsed.markdown,
+        assets: parsed.assets,
+        metadataJson: `${JSON.stringify(metadata, null, 2)}\n`,
+        mode: input.reparse ? "replace" : "create"
+      });
+
+      return ok({
+        document_id: input.documentId,
+        version: metadata.version,
+        raw_markdown_path: relativeRawMarkdownPath,
+        metadata_path: relativeMetadataPath,
+        status: "parsed",
+        warnings: parsed.warnings
+      });
+    } catch (error) {
+      return err(toWikiError(error, "validation_failed"));
+    } finally {
+      await lock?.release();
+    }
+  }
+
+  private async resolveParserRegistry(): Promise<DocumentParserRegistry> {
+    if (this.parserRegistry) {
+      return this.parserRegistry;
+    }
+
+    const config = await loadConfig(this.paths);
+    return new DocumentParserRegistry(config.documents);
+  }
+
+  private async readPendingIfExists(documentId: string, sessionId: string): Promise<PendingDocumentMetadata | undefined> {
+    try {
+      const raw = await readFile(this.documents.pendingPath(documentId), "utf8");
+      const metadata = JSON.parse(raw) as PendingDocumentMetadata;
+      if (!isPendingDocumentMetadata(metadata, documentId)) {
+        throw new WikiError("validation_failed", `Pending document metadata is invalid: ${documentId}`);
+      }
+
+      if (metadata.sessionId !== sessionId) {
+        return undefined;
+      }
+
+      return metadata;
+    } catch (error) {
+      if (isEnoent(error)) {
+        return undefined;
+      }
+
+      throw error;
+    }
+  }
+
+  private async readExistingSourceMetadataIfExists(worktreeRoot: string, documentId: string): Promise<SourceMetadata | undefined> {
+    try {
+      const raw = await readFile(sourceMetadataPath(worktreeRoot, documentId), "utf8");
+      const metadata = parseSourceMetadata(JSON.parse(raw));
+      const expectedRawMarkdownPath = sourceRelativePath(documentId, "raw.md");
+
+      if (metadata.documentId !== documentId || metadata.rawMarkdownPath !== expectedRawMarkdownPath) {
+        throw new WikiError("validation_failed", `Existing source metadata does not match document: ${documentId}`, {
+          documentId: metadata.documentId,
+          rawMarkdownPath: metadata.rawMarkdownPath,
+          expectedDocumentId: documentId,
+          expectedRawMarkdownPath
+        });
+      }
+
+      return metadata;
+    } catch (error) {
+      if (isEnoent(error)) {
+        return undefined;
+      }
+
+      throw error;
+    }
+  }
+
+  private async readBlob(blobSha256: string): Promise<Buffer> {
+    try {
+      return await readFile(this.documents.blobPath(blobSha256));
+    } catch (error) {
+      if (isEnoent(error)) {
+        throw new WikiError("blob_not_found", `Document blob not found: ${blobSha256}`, { blobSha256 });
+      }
+
+      throw error;
+    }
+  }
+
+  private async sessionWorktreeRoot(metadata: SessionMetadata): Promise<string> {
+    const absolute = path.resolve(this.paths.workspaceRoot, metadata.worktree);
+    const sessionsRoot = path.resolve(this.paths.sessionsWorktreePath);
+
+    if (!absolute.startsWith(`${sessionsRoot}${path.sep}`)) {
+      throw new WikiError("session_metadata_invalid", `Session worktree escapes sessions root: ${metadata.sessionId}`);
+    }
+
+    const realWorkspaceRoot = await realpath(this.paths.workspaceRoot);
+    await ensureDirectoryForWrite(this.paths.worktreesPath, realWorkspaceRoot, "Worktrees boundary");
+    const realWorktreesRoot = await realpath(this.paths.worktreesPath);
+
+    await ensureDirectoryForWrite(this.paths.sessionsWorktreePath, realWorktreesRoot, "Sessions worktree boundary");
+    const realSessionsRoot = await realpath(this.paths.sessionsWorktreePath);
+
+    await ensureDescendantDirectoryForWrite(this.paths.sessionsWorktreePath, absolute, realSessionsRoot, "Session worktree root");
+    const realWorktreeRoot = await realpath(absolute);
+    ensureStrictDescendant(realWorktreesRoot, realSessionsRoot, this.paths.sessionsWorktreePath, "Sessions worktree boundary escapes worktrees boundary");
+    ensureStrictDescendant(realSessionsRoot, realWorktreeRoot, absolute, "Session worktree root escapes sessions boundary");
+    return realWorktreeRoot;
+  }
+}
+
+function sourceTypeForMimeType(mimeType: string): DocumentSourceType {
+  if (mimeType === "application/pdf") {
+    return "pdf";
+  }
+
+  if (mimeType === "text/markdown" || mimeType === "text/x-markdown") {
+    return "markdown";
+  }
+
+  if (mimeType === "text/plain") {
+    return "text";
+  }
+
+  throw new WikiError("parser_not_supported", `Unsupported document MIME type: ${mimeType}`, { mimeType });
+}
+
+function decodeBase64(value: string): Buffer {
+  if (!isStrictBase64(value)) {
+    throw new WikiError("validation_failed", "Document content is not valid base64");
+  }
+
+  return Buffer.from(value, "base64");
+}
+
+function isStrictBase64(value: string): boolean {
+  return value.length % 4 === 0 && /^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test(value);
+}
+
+function sourceRelativePath(documentId: string, fileName: "metadata.json" | "raw.md"): string {
+  return `sources/${documentId}/${fileName}`;
+}
+
+type ParseLock = {
+  release(): Promise<void>;
+};
+
+const parseLockTtlMs = 30 * 60 * 1000;
+const parseLockHeartbeatIntervalMs = 60 * 1000;
+const parseLockAdminTtlMs = 60 * 1000;
+const parseLockReleaseRetryTimeoutMs = 1000;
+const parseLockReleaseRetryDelayMs = 10;
+
+type ParseLockMetadata = {
+  token: string;
+  createdAt: string;
+  updatedAt?: string;
+};
+
+async function acquireParseLock(paths: WorkspacePaths, sessionId: string, documentId: string): Promise<ParseLock> {
+  const locksRoot = path.join(paths.documentsStatePath, "parse-locks");
+  await mkdir(locksRoot, { recursive: true });
+  const lockPath = path.join(locksRoot, `${sessionId}--${documentId}.lock`);
+  const token = randomBytes(16).toString("hex");
+
+  await withParseLockAdminGuard(lockPath, async () => {
+    const existing = await readParseLockFreshness(lockPath);
+    if (existing) {
+      if (!isParseLockStale(existing)) {
+        throw new WikiError("validation_failed", `Document parse is already in progress: ${documentId}`, {
+          sessionId,
+          documentId
+        });
+      }
+
+      await rm(lockPath, { recursive: true, force: true });
+    }
+
+    await mkdir(lockPath);
+    try {
+      const now = new Date().toISOString();
+      await writeParseLockMetadata(lockPath, { token, createdAt: now, updatedAt: now });
+    } catch (error) {
+      await rm(lockPath, { recursive: true, force: true });
+      throw error;
+    }
+  });
+
+  const heartbeatTimer = startParseLockHeartbeat(lockPath, token);
+  return {
+    async release() {
+      stopParseLockHeartbeat(heartbeatTimer);
+      await releaseParseLockWithRetry(lockPath, token);
+    }
+  };
+}
+
+async function writeParseLockMetadata(lockPath: string, metadata: ParseLockMetadata): Promise<void> {
+  await writeFile(path.join(lockPath, "metadata.json"), `${JSON.stringify(metadata, null, 2)}\n`, "utf8");
+}
+
+function startParseLockHeartbeat(lockPath: string, token: string): ReturnType<typeof setInterval> {
+  const timer = setInterval(() => {
+    void updateParseLockHeartbeat(lockPath, token);
+  }, parseLockHeartbeatIntervalMs);
+  if (typeof timer === "object" && "unref" in timer && typeof timer.unref === "function") {
+    timer.unref();
+  }
+  return timer;
+}
+
+function stopParseLockHeartbeat(timer: ReturnType<typeof setInterval>): void {
+  clearInterval(timer);
+}
+
+async function updateParseLockHeartbeat(lockPath: string, token: string): Promise<void> {
+  await withParseLockAdminGuard(lockPath, async () => {
+    const metadata = await readParseLockMetadataIfExists(lockPath);
+    if (metadata?.token !== token) {
+      return;
+    }
+
+    await writeParseLockMetadata(lockPath, {
+      ...metadata,
+      updatedAt: new Date().toISOString()
+    });
+  }).catch(error => {
+    if (!isEnoent(error) && !isWikiErrorCode(error, "validation_failed")) {
+      throw error;
+    }
+  });
+}
+
+async function releaseParseLock(lockPath: string, token: string): Promise<void> {
+  const metadata = await readParseLockMetadataIfExists(lockPath);
+  if (metadata?.token !== token) {
+    return;
+  }
+
+  await rm(lockPath, { recursive: true, force: true });
+}
+
+async function releaseParseLockWithRetry(lockPath: string, token: string): Promise<void> {
+  const deadline = Date.now() + parseLockReleaseRetryTimeoutMs;
+  let lastError: unknown;
+
+  do {
+    try {
+      await withParseLockAdminGuard(lockPath, async () => {
+        await releaseParseLock(lockPath, token);
+      });
+      return;
+    } catch (error) {
+      if (!isWikiErrorCode(error, "validation_failed")) {
+        throw error;
+      }
+
+      lastError = error;
+      await delay(parseLockReleaseRetryDelayMs);
+    }
+  } while (Date.now() < deadline);
+
+  throw lastError;
+}
+
+async function withParseLockAdminGuard<T>(lockPath: string, callback: () => Promise<T>): Promise<T> {
+  const guardPath = `${lockPath}.admin`;
+  await acquireParseLockAdminGuard(guardPath);
+
+  try {
+    return await callback();
+  } finally {
+    await rm(guardPath, { recursive: true, force: true });
+  }
+}
+
+async function acquireParseLockAdminGuard(guardPath: string): Promise<void> {
+  for (let attempt = 0; attempt < 3; attempt += 1) {
+    try {
+      await mkdir(guardPath);
+      return;
+    } catch (error) {
+      if (!isFileExists(error)) {
+        throw error;
+      }
+
+      const guardStats = await lstatIfExists(guardPath);
+      if (!guardStats) {
+        continue;
+      }
+
+      if (Date.now() - guardStats.mtimeMs > parseLockAdminTtlMs) {
+        await rm(guardPath, { recursive: true, force: true });
+        continue;
+      }
+
+      throw new WikiError("validation_failed", `Document parse lock administration is already in progress: ${path.basename(guardPath)}`);
+    }
+  }
+
+  throw new WikiError("validation_failed", `Document parse lock administration is already in progress: ${path.basename(guardPath)}`);
+}
+
+type ParseLockFreshness = {
+  observedAtMs: number;
+};
+
+async function readParseLockFreshness(lockPath: string): Promise<ParseLockFreshness | undefined> {
+  const metadata = await readParseLockMetadataIfExists(lockPath);
+  const metadataTime = metadata ? Date.parse(metadata.updatedAt ?? metadata.createdAt) : Number.NaN;
+  if (!Number.isNaN(metadataTime)) {
+    return { observedAtMs: metadataTime };
+  }
+
+  const stats = await lstatIfExists(lockPath);
+  return stats ? { observedAtMs: stats.mtimeMs } : undefined;
+}
+
+function isParseLockStale(freshness: ParseLockFreshness): boolean {
+  return Date.now() - freshness.observedAtMs > parseLockTtlMs;
+}
+
+async function readParseLockMetadataIfExists(lockPath: string): Promise<ParseLockMetadata | undefined> {
+  let raw;
+  try {
+    raw = await readFile(path.join(lockPath, "metadata.json"), "utf8");
+  } catch (error) {
+    if (isEnoent(error)) {
+      return undefined;
+    }
+
+    throw error;
+  }
+
+  try {
+    const parsed = JSON.parse(raw) as Partial<ParseLockMetadata>;
+    if (typeof parsed.token !== "string" || typeof parsed.createdAt !== "string") {
+      return undefined;
+    }
+
+    return {
+      token: parsed.token,
+      createdAt: parsed.createdAt,
+      updatedAt: typeof parsed.updatedAt === "string" ? parsed.updatedAt : undefined
+    };
+  } catch {
+    return undefined;
+  }
+}
+
+async function lstatIfExists(filePath: string): Promise<{ mtimeMs: number } | undefined> {
+  try {
+    return await lstat(filePath);
+  } catch (error) {
+    if (isEnoent(error)) {
+      return undefined;
+    }
+
+    throw error;
+  }
+}
+
+async function ensureDescendantDirectoryForWrite(root: string, target: string, realBoundary: string, label: string): Promise<void> {
+  const relativePath = path.relative(root, target);
+  if (relativePath.length === 0 || relativePath.startsWith("..") || path.isAbsolute(relativePath)) {
+    throw new WikiError("invalid_path", `${label} escapes boundary: ${target}`);
+  }
+
+  const segments = relativePath.split(path.sep).filter(segment => segment.length > 0 && segment !== ".");
+  let current = root;
+
+  for (const segment of segments) {
+    current = path.join(current, segment);
+    await ensureDirectoryForWrite(current, realBoundary, label);
+  }
+}
+
+async function ensureDirectoryForWrite(directoryPath: string, realBoundary: string, label: string): Promise<void> {
+  try {
+    const stat = await lstat(directoryPath);
+
+    if (stat.isSymbolicLink()) {
+      throw new WikiError("invalid_path", `${label} is a symlink: ${directoryPath}`);
+    }
+
+    if (!stat.isDirectory()) {
+      throw new WikiError("invalid_path", `${label} is not a directory: ${directoryPath}`);
+    }
+  } catch (error) {
+    if (!isEnoent(error)) {
+      throw error;
+    }
+
+    await mkdir(directoryPath);
+    await ensureDirectoryNotSymlink(directoryPath, label);
+  }
+
+  const realDirectory = await realpath(directoryPath);
+  ensureInsideRealRoot(realBoundary, realDirectory, directoryPath, `${label} escapes boundary`);
+}
+
+async function ensureDirectoryNotSymlink(directoryPath: string, label: string): Promise<void> {
+  const stat = await lstat(directoryPath);
+
+  if (stat.isSymbolicLink()) {
+    throw new WikiError("invalid_path", `${label} is a symlink: ${directoryPath}`);
+  }
+
+  if (!stat.isDirectory()) {
+    throw new WikiError("invalid_path", `${label} is not a directory: ${directoryPath}`);
+  }
+}
+
+function ensureStrictDescendant(realRoot: string, target: string, originalPath: string, message: string): void {
+  if (!target.startsWith(`${realRoot}${path.sep}`)) {
+    throw new WikiError("invalid_path", `${message}: ${originalPath}`);
+  }
+}
+
+function ensureInsideRealRoot(realRoot: string, target: string, originalPath: string, message: string): void {
+  if (!target.startsWith(`${realRoot}${path.sep}`) && target !== realRoot) {
+    throw new WikiError("invalid_path", `${message}: ${originalPath}`);
+  }
+}
+
+function isPendingDocumentMetadata(value: unknown, documentId: string): value is PendingDocumentMetadata {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+
+  const metadata = value as Partial<PendingDocumentMetadata>;
+  return (
+    metadata.documentId === documentId &&
+    typeof metadata.sessionId === "string" &&
+    metadata.sessionId.length > 0 &&
+    isSupportedPendingSource(metadata.sourceType, metadata.mimeType) &&
+    typeof metadata.originalFileName === "string" &&
+    metadata.originalFileName.trim().length > 0 &&
+    typeof metadata.blobSha256 === "string" &&
+    /^[0-9a-f]{64}$/.test(metadata.blobSha256) &&
+    typeof metadata.createdAt === "string" &&
+    typeof metadata.updatedAt === "string" &&
+    (metadata.createdBy === undefined || typeof metadata.createdBy === "string") &&
+    metadata.status === "uploaded"
+  );
+}
+
+function isSupportedPendingSource(sourceType: unknown, mimeType: unknown): sourceType is DocumentSourceType {
+  return (
+    (sourceType === "pdf" && mimeType === "application/pdf") ||
+    (sourceType === "markdown" && (mimeType === "text/markdown" || mimeType === "text/x-markdown")) ||
+    (sourceType === "text" && mimeType === "text/plain")
+  );
+}
+
+function isEnoent(error: unknown): boolean {
+  return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
+}
+
+function isFileExists(error: unknown): boolean {
+  return Boolean(error && typeof error === "object" && "code" in error && error.code === "EEXIST");
+}
+
+function isWikiErrorCode(error: unknown, code: string): boolean {
+  return error instanceof WikiError && error.code === code;
+}
+
+async function delay(milliseconds: number): Promise<void> {
+  await new Promise(resolve => setTimeout(resolve, milliseconds));
+}
+
+export const __documentServiceTest = {
+  acquireParseLock
+};