tgo-wiki 0.1.0

package/packages/core/src/documents/document-store.ts

@@ -0,0 +1,132 @@
+import { createHash, randomBytes } from "node:crypto";
+import { link, mkdir, readFile, unlink, writeFile } from "node:fs/promises";
+import path from "node:path";
+import type { WorkspacePaths } from "../config/workspace-resolver.js";
+import { WikiError } from "../errors.js";
+import { assertValidDocumentId } from "./document-id.js";
+import type { PendingDocumentMetadata } from "./document-types.js";
+
+export class DocumentStore {
+  constructor(private readonly paths: WorkspacePaths) {}
+
+  async writeBlobIfAbsent(blobSha256: string, content: Buffer): Promise<void> {
+    assertValidSha256(blobSha256);
+    assertContentMatchesSha256(content, blobSha256);
+    await mkdir(this.paths.documentBlobsPath, { recursive: true });
+    const blobPath = this.blobPath(blobSha256);
+
+    try {
+      await writeAtomicallyNoOverwrite(this.paths.documentBlobsPath, blobPath, content);
+    } catch (error) {
+      if (isFileExists(error)) {
+        await assertExistingBlobMatches(blobPath, blobSha256);
+        return;
+      }
+
+      throw error;
+    }
+  }
+
+  async writePending(metadata: PendingDocumentMetadata): Promise<void> {
+    assertValidDocumentId(metadata.documentId);
+    await mkdir(this.paths.pendingDocumentsPath, { recursive: true });
+    const pendingPath = this.pendingPath(metadata.documentId);
+    const content = Buffer.from(`${JSON.stringify(metadata, null, 2)}\n`, "utf8");
+
+    try {
+      await writeAtomicallyNoOverwrite(this.paths.pendingDocumentsPath, pendingPath, content);
+    } catch (error) {
+      if (isFileExists(error)) {
+        throw new WikiError("source_exists", `Pending document metadata already exists: ${metadata.documentId}`);
+      }
+
+      throw error;
+    }
+  }
+
+  blobPath(blobSha256: string): string {
+    assertValidSha256(blobSha256);
+    return path.join(this.paths.documentBlobsPath, blobSha256);
+  }
+
+  pendingPath(documentId: string): string {
+    assertValidDocumentId(documentId);
+    return path.join(this.paths.pendingDocumentsPath, `${documentId}.json`);
+  }
+}
+
+function assertValidSha256(value: string): void {
+  if (!/^[0-9a-f]{64}$/.test(value)) {
+    throw new WikiError("validation_failed", `Invalid sha256: ${value}`);
+  }
+}
+
+function assertContentMatchesSha256(content: Buffer, expected: string): void {
+  const actual = createHash("sha256").update(content).digest("hex");
+
+  if (actual !== expected) {
+    throw new WikiError("validation_failed", `Blob content does not match sha256: ${expected}`, {
+      expected,
+      actual
+    });
+  }
+}
+
+async function writeAtomicallyNoOverwrite(directory: string, finalPath: string, content: Buffer): Promise<void> {
+  const tempPath = await writeUniqueTempFile(directory, path.basename(finalPath), content);
+
+  try {
+    await link(tempPath, finalPath);
+  } finally {
+    await unlinkIfExists(tempPath);
+  }
+}
+
+async function writeUniqueTempFile(directory: string, finalBaseName: string, content: Buffer): Promise<string> {
+  for (let attempt = 0; attempt < 10; attempt += 1) {
+    const tempPath = path.join(directory, `.${finalBaseName}.${randomBytes(8).toString("hex")}.tmp`);
+
+    try {
+      await writeFile(tempPath, content, { flag: "wx" });
+      return tempPath;
+    } catch (error) {
+      if (isFileExists(error)) {
+        continue;
+      }
+
+      throw error;
+    }
+  }
+
+  throw new WikiError("validation_failed", `Could not create unique temp file for: ${finalBaseName}`);
+}
+
+async function assertExistingBlobMatches(blobPath: string, blobSha256: string): Promise<void> {
+  const existing = await readFile(blobPath);
+  const actual = createHash("sha256").update(existing).digest("hex");
+
+  if (actual !== blobSha256) {
+    throw new WikiError("validation_failed", `Existing blob content does not match sha256 file name: ${blobSha256}`, {
+      expected: blobSha256,
+      actual
+    });
+  }
+}
+
+async function unlinkIfExists(filePath: string): Promise<void> {
+  try {
+    await unlink(filePath);
+  } catch (error) {
+    if (!isEnoent(error)) {
+      throw error;
+    }
+  }
+}
+
+function isFileExists(error: unknown): boolean {
+  return Boolean(error && typeof error === "object" && "code" in error && error.code === "EEXIST");
+}
+
+function isEnoent(error: unknown): boolean {
+  return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
+}

package/packages/core/src/documents/document-types.ts

@@ -0,0 +1,33 @@
+export type DocumentIdGenerator = () => string;
+
+export type DocumentSourceType = "pdf" | "markdown" | "text";
+
+export type PendingDocumentStatus = "uploaded";
+
+export type DocumentUploadInput = {
+  sessionId: string;
+  fileName: string;
+  mimeType: string;
+  contentBase64: string;
+  createdBy?: string;
+};
+
+export type DocumentUploadResult = {
+  document_id: string;
+  blob_sha256: string;
+  size: number;
+  status: PendingDocumentStatus;
+};
+
+export type PendingDocumentMetadata = {
+  documentId: string;
+  sessionId: string;
+  sourceType: DocumentSourceType;
+  originalFileName: string;
+  mimeType: string;
+  blobSha256: string;
+  createdAt: string;
+  updatedAt: string;
+  createdBy?: string;
+  status: PendingDocumentStatus;
+};

package/packages/core/src/documents/pdf-text-parser.ts

@@ -0,0 +1,35 @@
+import { PDFParse } from "pdf-parse";
+import { WikiError, toWikiError } from "../errors.js";
+import type { DocumentParseInput, DocumentParseOutput, DocumentParser } from "./document-parser-registry.js";
+
+export class PdfTextParser implements DocumentParser {
+  readonly name = "pdf-text";
+  readonly version = "1.0.0";
+  readonly supportedMimeTypes = ["application/pdf"];
+
+  async parse(input: DocumentParseInput): Promise<DocumentParseOutput> {
+    try {
+      const pdf = new PDFParse({ data: input.content });
+      try {
+        const extracted = await pdf.getText();
+        const text = extracted.text.trim();
+
+        return {
+          markdown: `# ${input.fileName}\n\n> Source document parsed from PDF.\n> Parser: pdf-text\n> Parsed at: ${input.parsedAt}\n\n${text}\n`,
+          assets: [],
+          warnings: [],
+          metadata: { pages: extracted.total }
+        };
+      } finally {
+        await pdf.destroy();
+      }
+    } catch (error) {
+      const wikiError = toWikiError(error, "parse_failed");
+      if (wikiError.code === "parse_failed") {
+        throw wikiError;
+      }
+
+      throw new WikiError("parse_failed", wikiError.message, wikiError.details);
+    }
+  }
+}

package/packages/core/src/documents/text-markdown-parser.ts

@@ -0,0 +1,50 @@
+import type { DocumentParseInput, DocumentParseOutput, DocumentParser } from "./document-parser-registry.js";
+
+export class TextMarkdownParser implements DocumentParser {
+  readonly name = "text-markdown";
+  readonly version = "1.0.0";
+  readonly supportedMimeTypes = ["text/markdown", "text/x-markdown", "text/plain"];
+
+  async parse(input: DocumentParseInput): Promise<DocumentParseOutput> {
+    const text = stripUtf8Bom(input.content.toString("utf8"));
+    const markdown = input.mimeType === "text/plain" ? plainTextMarkdown(input.fileName, text, input.parsedAt) : ensureFinalNewline(text);
+
+    return {
+      markdown,
+      assets: [],
+      warnings: [],
+      metadata: {
+        characters: text.length,
+        lines: countLines(text)
+      }
+    };
+  }
+}
+
+function plainTextMarkdown(fileName: string, text: string, parsedAt: string): string {
+  return [
+    `# ${fileName}`,
+    "",
+    "> Source document parsed from plain text.",
+    "> Parser: text-markdown",
+    `> Parsed at: ${parsedAt}`,
+    "",
+    ensureFinalNewline(text).trimEnd()
+  ].join("\n") + "\n";
+}
+
+function ensureFinalNewline(value: string): string {
+  return value.endsWith("\n") ? value : `${value}\n`;
+}
+
+function stripUtf8Bom(value: string): string {
+  return value.startsWith("\uFEFF") ? value.slice(1) : value;
+}
+
+function countLines(value: string): number {
+  if (value.length === 0) {
+    return 0;
+  }
+
+  return value.endsWith("\n") ? value.slice(0, -1).split("\n").length : value.split("\n").length;
+}

package/packages/core/src/errors.ts

@@ -0,0 +1,46 @@
+export type WikiErrorCode =
+  | "permission_denied"
+  | "invalid_path"
+  | "invalid_frontmatter"
+  | "validation_failed"
+  | "session_not_found"
+  | "session_metadata_invalid"
+  | "session_dirty"
+  | "needs_rebase"
+  | "unsupported_channel"
+  | "git_error"
+  | "vfs_command_denied"
+  | "document_not_found"
+  | "blob_not_found"
+  | "parser_not_supported"
+  | "parse_failed"
+  | "invalid_url"
+  | "fetch_failed"
+  | "fetch_timeout"
+  | "response_too_large"
+  | "source_exists"
+  | "source_write_denied";
+
+export class WikiError extends Error {
+  readonly code: WikiErrorCode;
+  readonly details: Record<string, unknown>;
+
+  constructor(code: WikiErrorCode, message: string, details: Record<string, unknown> = {}) {
+    super(message);
+    this.name = "WikiError";
+    this.code = code;
+    this.details = details;
+  }
+}
+
+export function toWikiError(error: unknown, fallbackCode: WikiErrorCode): WikiError {
+  if (error instanceof WikiError) {
+    return error;
+  }
+
+  if (error instanceof Error) {
+    return new WikiError(fallbackCode, error.message);
+  }
+
+  return new WikiError(fallbackCode, String(error));
+}

package/packages/core/src/git/git-service.ts

@@ -0,0 +1,68 @@
+import { spawn } from "node:child_process";
+import { WikiError } from "../errors.js";
+
+export type GitRunResult = {
+  stdout: string;
+  stderr: string;
+  exitCode: number;
+};
+
+export class GitService {
+  async run(args: string[], options: { cwd: string }): Promise<GitRunResult> {
+    return new Promise((resolve, reject) => {
+      const child = spawn("git", args, {
+        cwd: options.cwd,
+        stdio: ["ignore", "pipe", "pipe"]
+      });
+
+      let stdout = "";
+      let stderr = "";
+
+      child.stdout.setEncoding("utf8");
+      child.stderr.setEncoding("utf8");
+      child.stdout.on("data", chunk => {
+        stdout += chunk;
+      });
+      child.stderr.on("data", chunk => {
+        stderr += chunk;
+      });
+      child.on("error", error => {
+        reject(new WikiError("git_error", error.message, { args }));
+      });
+      child.on("close", exitCode => {
+        const code = exitCode ?? 1;
+        const result = { stdout, stderr, exitCode: code };
+
+        if (code !== 0) {
+          reject(
+            new WikiError("git_error", stderr.trim() || stdout.trim() || "git command failed", {
+              args,
+              exitCode: code
+            })
+          );
+          return;
+        }
+
+        resolve(result);
+      });
+    });
+  }
+
+  async commit(cwd: string, message: string, pathspecs = ["wiki"]): Promise<string> {
+    await this.run(["add", "--", ...stagingPathspecs(pathspecs)], { cwd });
+    await this.run(
+      ["-c", "user.name=TGO Wiki", "-c", "user.email=tgo-wiki@example.invalid", "commit", "-m", message],
+      { cwd }
+    );
+    const rev = await this.run(["rev-parse", "HEAD"], { cwd });
+    return rev.stdout.trim();
+  }
+}
+
+function stagingPathspecs(pathspecs: string[]): string[] {
+  if (pathspecs.length === 1 && pathspecs[0] === "wiki") {
+    return pathspecs;
+  }
+
+  return pathspecs.map(pathspec => `:(literal)${pathspec}`);
+}

package/packages/core/src/index.ts

@@ -0,0 +1,38 @@
+export * from "./errors.js";
+export * from "./result.js";
+export * from "./config/defaults.js";
+export * from "./config/workspace-resolver.js";
+export * from "./config/config-loader.js";
+export * from "./git/git-service.js";
+export * from "./services/workspace-service.js";
+export * from "./services/wiki-service.js";
+export * from "./services/session-workflow-service.js";
+export * from "./validation/path-validator.js";
+export * from "./validation/frontmatter-validator.js";
+export * from "./validation/validation-service.js";
+export * from "./session/session-types.js";
+export * from "./session/session-id.js";
+export * from "./session/session-store.js";
+export * from "./session/session-service.js";
+export * from "./vfs/vfs-command-parser.js";
+export * from "./vfs/vfs-service.js";
+export * from "./permissions/permission-service.js";
+export * from "./publish/publish-service.js";
+export * from "./sources/source-id.js";
+export * from "./sources/source-paths.js";
+export * from "./sources/source-types.js";
+export * from "./sources/source-validator.js";
+export * from "./sources/source-service.js";
+export * from "./documents/document-id.js";
+export * from "./documents/document-types.js";
+export * from "./documents/document-store.js";
+export * from "./documents/document-service.js";
+export * from "./documents/command-document-parser.js";
+export * from "./documents/pdf-text-parser.js";
+export * from "./documents/text-markdown-parser.js";
+export * from "./web/web-types.js";
+export * from "./web/web-id.js";
+export * from "./web/web-paths.js";
+export * from "./web/html-to-markdown.js";
+export * from "./web/static-web-fetcher.js";
+export * from "./web/web-ingestion-service.js";

package/packages/core/src/markdown/markdown-scanner.ts

@@ -0,0 +1,90 @@
+export type MarkdownLink = {
+  kind: "markdown" | "wiki";
+  target: string;
+  raw: string;
+};
+
+export function firstH1(content: string): string | undefined {
+  for (const line of nonFencedLines(content)) {
+    const match = /^#\s+(.+?)\s*$/.exec(line);
+
+    if (match) {
+      return match[1]?.replace(/\s+#+$/, "").trim();
+    }
+  }
+
+  return undefined;
+}
+
+export function findMarkdownLinks(content: string): MarkdownLink[] {
+  const links: MarkdownLink[] = [];
+  const markdownLinkPattern = /(?<!!)\[[^\]]+\]\(([^)\s]+)(?:\s+"[^"]*")?\)/g;
+  const wikiLinkPattern = /\[\[([^\]|#]+)(?:#[^\]|]+)?(?:\|[^\]]+)?\]\]/g;
+  const searchableContent = nonFencedLines(content).join("\n");
+
+  for (const match of searchableContent.matchAll(markdownLinkPattern)) {
+    const raw = match[0];
+    const target = match[1]?.trim();
+
+    if (!target || isIgnoredMarkdownTarget(target) || isEscaped(searchableContent, match.index ?? 0)) {
+      continue;
+    }
+
+    links.push({ kind: "markdown", target, raw });
+  }
+
+  for (const match of searchableContent.matchAll(wikiLinkPattern)) {
+    const raw = match[0];
+    const target = normalizeWikiLinkTarget(match[1] ?? "");
+
+    if (!target || isEscaped(searchableContent, match.index ?? 0)) {
+      continue;
+    }
+
+    links.push({ kind: "wiki", target, raw });
+  }
+
+  return links;
+}
+
+function isIgnoredMarkdownTarget(target: string): boolean {
+  return target.startsWith("#") || /^https?:\/\//i.test(target);
+}
+
+function normalizeWikiLinkTarget(target: string): string | undefined {
+  const normalized = target.trim().replaceAll("\\", "/").replace(/^\/+/, "");
+
+  if (!normalized) {
+    return undefined;
+  }
+
+  return normalized.endsWith(".md") ? `wiki/${normalized}` : `wiki/${normalized}.md`;
+}
+
+function nonFencedLines(content: string): string[] {
+  const lines: string[] = [];
+  let inFence = false;
+
+  for (const line of content.split(/\r?\n/)) {
+    if (/^\s*(```|~~~)/.test(line)) {
+      inFence = !inFence;
+      continue;
+    }
+
+    if (!inFence) {
+      lines.push(line);
+    }
+  }
+
+  return lines;
+}
+
+function isEscaped(content: string, index: number): boolean {
+  let slashCount = 0;
+
+  for (let cursor = index - 1; cursor >= 0 && content[cursor] === "\\"; cursor -= 1) {
+    slashCount += 1;
+  }
+
+  return slashCount % 2 === 1;
+}

package/packages/core/src/permissions/permission-service.ts

@@ -0,0 +1,50 @@
+import { WikiError } from "../errors.js";
+
+export type Role = "reader" | "writer" | "publisher" | "admin";
+
+export type ToolName =
+  | "vfs_exec"
+  | "wiki_read"
+  | "wiki_channel_status"
+  | "wiki_session_start"
+  | "wiki_session_patch"
+  | "wiki_session_validate"
+  | "wiki_session_diff"
+  | "wiki_session_commit"
+  | "document_upload"
+  | "document_parse"
+  | "web_fetch"
+  | "source_list"
+  | "source_read"
+  | "wiki_publish_session";
+
+const readerTools = new Set<ToolName>(["vfs_exec", "wiki_read", "wiki_channel_status", "source_list", "source_read"]);
+const writerTools = new Set<ToolName>([
+  ...readerTools,
+  "wiki_session_start",
+  "wiki_session_patch",
+  "wiki_session_validate",
+  "wiki_session_diff",
+  "wiki_session_commit",
+  "document_upload",
+  "document_parse"
+]);
+const publisherTools = new Set<ToolName>([...writerTools, "web_fetch", "wiki_publish_session"]);
+
+export function assertCanUseTool(role: Role, toolName: ToolName): void {
+  const allowed =
+    role === "reader"
+      ? readerTools
+      : role === "writer"
+        ? writerTools
+        : role === "publisher" || role === "admin"
+          ? publisherTools
+          : null;
+  if (allowed === null) {
+    throw new WikiError("permission_denied", `${role} cannot use ${toolName}`);
+  }
+
+  if (!allowed.has(toolName)) {
+    throw new WikiError("permission_denied", `${role} cannot use ${toolName}`);
+  }
+}

package/packages/core/src/publish/publish-service.ts

@@ -0,0 +1,142 @@
+import path from "node:path";
+import { resolveWorkspacePaths, type WorkspacePaths } from "../config/workspace-resolver.js";
+import { WikiError, toWikiError } from "../errors.js";
+import { GitService } from "../git/git-service.js";
+import { err, ok, type Result } from "../result.js";
+import { SessionWorkflowService } from "../services/session-workflow-service.js";
+import { SessionStore } from "../session/session-store.js";
+import type { SessionMetadata } from "../session/session-types.js";
+
+export type PublishInput = {
+  sessionId: string;
+  channel?: "stable" | string;
+  mode?: "ff-only" | string;
+};
+
+export type PublishResult = {
+  ok: true;
+  channel: "stable";
+  previous_commit: string;
+  published_commit: string;
+};
+
+export class PublishService {
+  private readonly paths: WorkspacePaths;
+  private readonly store: SessionStore;
+  private readonly workflow: SessionWorkflowService;
+
+  constructor(
+    workspaceRoot: string,
+    private readonly git = new GitService()
+  ) {
+    this.paths = resolveWorkspacePaths(workspaceRoot);
+    this.store = new SessionStore(this.paths);
+    this.workflow = new SessionWorkflowService(workspaceRoot, undefined, git);
+  }
+
+  async publish(input: PublishInput): Promise<Result<PublishResult>> {
+    try {
+      if ((input.channel ?? "stable") !== "stable") {
+        return err(new WikiError("unsupported_channel", `Unsupported publish channel: ${input.channel}`));
+      }
+
+      if ((input.mode ?? "ff-only") !== "ff-only") {
+        return err(new WikiError("unsupported_channel", `Unsupported publish mode: ${input.mode}`));
+      }
+
+      const metadata = await this.store.read(input.sessionId);
+
+      if (metadata.status !== "open") {
+        return err(new WikiError("session_dirty", `Session is not open: ${input.sessionId}`, { status: metadata.status }));
+      }
+
+      if (!metadata.lastCommit) {
+        return err(new WikiError("session_dirty", `Session has no committed changes: ${input.sessionId}`));
+      }
+
+      const sessionWorktree = this.sessionWorktreeRoot(metadata);
+      const status = await this.git.run(["status", "--porcelain"], { cwd: sessionWorktree });
+
+      if (status.stdout.trim().length > 0) {
+        return err(new WikiError("session_dirty", "Session has uncommitted changes"));
+      }
+
+      const branchHead = (await this.git.run(["rev-parse", metadata.branch], { cwd: this.paths.repoPath })).stdout.trim();
+
+      if (branchHead !== metadata.lastCommit) {
+        return err(
+          new WikiError("session_dirty", "Session branch head differs from last committed session metadata", {
+            branch: metadata.branch,
+            branchHead,
+            lastCommit: metadata.lastCommit
+          })
+        );
+      }
+
+      const validation = await this.workflow.validate({ sessionId: input.sessionId });
+
+      if (!validation.ok) {
+        return validation;
+      }
+
+      if (!validation.value.ok) {
+        return err(new WikiError("validation_failed", "Session validation failed", { validation: validation.value }));
+      }
+
+      const stableStatus = await this.git.run(["status", "--porcelain"], { cwd: this.paths.stableWorktreePath });
+
+      if (stableStatus.stdout.trim().length > 0) {
+        return err(new WikiError("session_dirty", "Stable worktree has uncommitted changes"));
+      }
+
+      const previousCommit = (await this.git.run(["rev-parse", "HEAD"], { cwd: this.paths.stableWorktreePath })).stdout.trim();
+
+      try {
+        await this.git.run(["merge", "--ff-only", metadata.lastCommit], { cwd: this.paths.stableWorktreePath });
+      } catch (error) {
+        if (isFastForwardFailure(error)) {
+          return err(new WikiError("needs_rebase", "Stable has advanced; rebase the session before publishing"));
+        }
+
+        throw error;
+      }
+
+      const publishedCommit = (await this.git.run(["rev-parse", "HEAD"], { cwd: this.paths.stableWorktreePath })).stdout.trim();
+
+      await this.git.run(["worktree", "remove", sessionWorktree], { cwd: this.paths.repoPath });
+
+      const now = new Date().toISOString();
+      await this.store.update(input.sessionId, current => ({
+        ...current,
+        status: "published",
+        publishedAt: now,
+        publishedCommit,
+        updatedAt: now
+      }));
+
+      return ok({
+        ok: true,
+        channel: "stable",
+        previous_commit: previousCommit,
+        published_commit: publishedCommit
+      });
+    } catch (error) {
+      return err(toWikiError(error, "git_error"));
+    }
+  }
+
+  private sessionWorktreeRoot(metadata: SessionMetadata): string {
+    const absolute = path.resolve(this.paths.workspaceRoot, metadata.worktree);
+    const sessionsRoot = path.resolve(this.paths.sessionsWorktreePath);
+
+    if (!absolute.startsWith(`${sessionsRoot}${path.sep}`) && absolute !== sessionsRoot) {
+      throw new WikiError("session_metadata_invalid", `Session worktree escapes sessions root: ${metadata.sessionId}`);
+    }
+
+    return absolute;
+  }
+}
+
+function isFastForwardFailure(error: unknown): boolean {
+  return error instanceof WikiError && /fast-forward|Not possible to fast-forward/i.test(error.message);
+}