tgo-wiki 0.1.0

package/packages/core/src/sources/source-service.ts

@@ -0,0 +1,416 @@
+import type { Dirent } from "node:fs";
+import { lstat, readFile, readdir, realpath } from "node:fs/promises";
+import path from "node:path";
+import { resolveWorkspacePaths, type WorkspacePaths } from "../config/workspace-resolver.js";
+import { WikiError } from "../errors.js";
+import { err, ok, type Result } from "../result.js";
+import { assertValidSessionId } from "../session/session-id.js";
+import { SessionStore } from "../session/session-store.js";
+import type { SessionMetadata } from "../session/session-types.js";
+import { assertValidSourceId } from "./source-id.js";
+import { sourceDirectory } from "./source-paths.js";
+import type { SourceListInput, SourceListResult, SourceMetadata, SourceReadInput, SourceReadResult } from "./source-types.js";
+import { parseSourceMetadata } from "./source-validator.js";
+
+type ResolvedSourceRef = {
+  ref: string;
+  worktreeRoot: string;
+};
+
+const unsupportedBareRefs = new Set(["draft"]);
+
+export class SourceService {
+  private readonly paths: WorkspacePaths;
+  private readonly store: SessionStore;
+
+  constructor(workspaceRoot: string) {
+    this.paths = resolveWorkspacePaths(workspaceRoot);
+    this.store = new SessionStore(this.paths);
+  }
+
+  async list(input: SourceListInput = {}): Promise<Result<SourceListResult>> {
+    try {
+      const resolved = await this.resolveRef(input.ref);
+      const sourcesRoot = path.join(resolved.worktreeRoot, "sources");
+      const entries = await readSourcesDirectory(sourcesRoot);
+      const sources: SourceListResult["sources"] = [];
+
+      for (const entry of entries) {
+        if (!entry.isDirectory() && !entry.isSymbolicLink()) {
+          continue;
+        }
+
+        const metadata = await this.readMetadataIfExists(resolved.worktreeRoot, entry.name);
+        if (metadata === undefined) {
+          continue;
+        }
+
+        sources.push(toSourceListItem(metadata));
+      }
+
+      return ok({ sources: sources.sort((a, b) => a.document_id.localeCompare(b.document_id)) });
+    } catch (error) {
+      return err(toSourceError(error, "document_not_found"));
+    }
+  }
+
+  async read(input: SourceReadInput): Promise<Result<SourceReadResult>> {
+    try {
+      assertValidSourceId(input.document_id);
+      const resolved = await this.resolveRef(input.ref);
+      const metadata = await this.readMetadata(resolved.worktreeRoot, input.document_id);
+      const rawMarkdown = await readRequiredSourceFile(resolved.worktreeRoot, input.document_id, "raw.md");
+
+      return ok({
+        document_id: input.document_id,
+        ref: resolved.ref,
+        metadata: metadata as unknown as Record<string, unknown>,
+        raw_markdown: rawMarkdown
+      });
+    } catch (error) {
+      return err(toSourceError(error, "document_not_found"));
+    }
+  }
+
+  private async readMetadata(worktreeRoot: string, documentId: string): Promise<SourceMetadata> {
+    assertValidSourceId(documentId);
+    const raw = await readRequiredSourceFile(worktreeRoot, documentId, "metadata.json");
+    return parseMetadataJson(raw, documentId);
+  }
+
+  private async readMetadataIfExists(worktreeRoot: string, documentId: string): Promise<SourceMetadata | undefined> {
+    assertValidSourceId(documentId);
+    const raw = await readOptionalSourceFile(worktreeRoot, documentId, "metadata.json");
+    if (raw === undefined) {
+      return undefined;
+    }
+
+    return parseMetadataJson(raw, documentId);
+  }
+
+  private async resolveRef(ref: SourceListInput["ref"]): Promise<ResolvedSourceRef> {
+    if (ref === undefined || ref === "stable") {
+      return {
+        ref: "stable",
+        worktreeRoot: await resolveSafeStableWorktreeRoot(this.paths)
+      };
+    }
+
+    if (ref.startsWith("wiki/session/")) {
+      const sessionId = ref.slice("wiki/session/".length);
+      const metadata = await this.store.read(sessionId);
+      return {
+        ref: `wiki/session/${metadata.sessionId}`,
+        worktreeRoot: await this.sessionWorktreeRoot(metadata)
+      };
+    }
+
+    if (unsupportedBareRefs.has(ref)) {
+      throw new WikiError("unsupported_channel", `Unsupported source ref: ${ref}`);
+    }
+
+    if (!isValidBareSessionRef(ref)) {
+      throw new WikiError("unsupported_channel", `Unsupported source ref: ${ref}`);
+    }
+
+    const metadata = await this.store.read(ref);
+    return {
+      ref: `wiki/session/${metadata.sessionId}`,
+      worktreeRoot: await this.sessionWorktreeRoot(metadata)
+    };
+  }
+
+  private async sessionWorktreeRoot(metadata: SessionMetadata): Promise<string> {
+    const absolute = path.resolve(this.paths.workspaceRoot, metadata.worktree);
+    const sessionsRoot = path.resolve(this.paths.sessionsWorktreePath);
+
+    if (!absolute.startsWith(`${sessionsRoot}${path.sep}`) && absolute !== sessionsRoot) {
+      throw new WikiError("session_metadata_invalid", `Session worktree escapes sessions root: ${metadata.sessionId}`);
+    }
+
+    return await resolveSafeSessionWorktreeRoot(absolute, this.paths);
+  }
+}
+
+async function readSourcesDirectory(sourcesRoot: string): Promise<Dirent[]> {
+  try {
+    const stat = await lstat(sourcesRoot);
+
+    if (stat.isSymbolicLink()) {
+      throw new WikiError("invalid_path", `Sources directory is a symlink: ${sourcesRoot}`);
+    }
+
+    if (!stat.isDirectory()) {
+      throw new WikiError("invalid_path", `Sources path is not a directory: ${sourcesRoot}`);
+    }
+
+    return await readdir(sourcesRoot, { withFileTypes: true });
+  } catch (error) {
+    if (isEnoent(error)) {
+      return [];
+    }
+
+    throw error;
+  }
+}
+
+async function resolveSafeStableWorktreeRoot(paths: WorkspacePaths): Promise<string> {
+  await ensureConfiguredDirectoryNotSymlink(paths.worktreesPath, "Worktrees boundary");
+  await ensureConfiguredDirectoryNotSymlink(paths.stableWorktreePath, "Stable worktree root");
+  const realWorktreesRoot = await realpath(paths.worktreesPath);
+  const realStableRoot = await realpath(paths.stableWorktreePath);
+  ensureStrictDescendant(realWorktreesRoot, realStableRoot, paths.stableWorktreePath, "Stable worktree root escapes worktrees boundary");
+  return realStableRoot;
+}
+
+async function resolveSafeSessionWorktreeRoot(worktreeRoot: string, paths: WorkspacePaths): Promise<string> {
+  await ensureConfiguredDirectoryNotSymlink(paths.worktreesPath, "Worktrees boundary");
+  await ensureConfiguredDirectoryNotSymlink(paths.sessionsWorktreePath, "Sessions worktree boundary");
+  await ensureConfiguredDirectoryNotSymlink(worktreeRoot, "Session worktree root");
+  const realWorktreesRoot = await realpath(paths.worktreesPath);
+  const realSessionsRoot = await realpath(paths.sessionsWorktreePath);
+  const realWorktreeRoot = await realpath(worktreeRoot);
+  ensureStrictDescendant(realWorktreesRoot, realSessionsRoot, paths.sessionsWorktreePath, "Sessions worktree boundary escapes worktrees boundary");
+  ensureStrictDescendant(realSessionsRoot, realWorktreeRoot, worktreeRoot, "Session worktree root escapes sessions boundary");
+  return realWorktreeRoot;
+}
+
+async function readRequiredSourceFile(worktreeRoot: string, documentId: string, fileName: "metadata.json" | "raw.md"): Promise<string> {
+  const filePath = await resolveSourceFileForRead(worktreeRoot, documentId, fileName, false);
+
+  try {
+    return await readFile(filePath, "utf8");
+  } catch (error) {
+    if (isEnoent(error)) {
+      throw new WikiError("document_not_found", `Source file not found: ${filePath}`);
+    }
+
+    throw error;
+  }
+}
+
+async function readOptionalSourceFile(
+  worktreeRoot: string,
+  documentId: string,
+  fileName: "metadata.json" | "raw.md"
+): Promise<string | undefined> {
+  const filePath = await resolveSourceFileForRead(worktreeRoot, documentId, fileName, true);
+  if (filePath === undefined) {
+    return undefined;
+  }
+
+  try {
+    return await readFile(filePath, "utf8");
+  } catch (error) {
+    if (isEnoent(error)) {
+      return undefined;
+    }
+
+    throw error;
+  }
+}
+
+async function resolveSourceFileForRead(
+  worktreeRoot: string,
+  documentId: string,
+  fileName: "metadata.json" | "raw.md",
+  optional: false
+): Promise<string>;
+async function resolveSourceFileForRead(
+  worktreeRoot: string,
+  documentId: string,
+  fileName: "metadata.json" | "raw.md",
+  optional: true
+): Promise<string | undefined>;
+async function resolveSourceFileForRead(
+  worktreeRoot: string,
+  documentId: string,
+  fileName: "metadata.json" | "raw.md",
+  optional: boolean
+): Promise<string | undefined> {
+  assertValidSourceId(documentId);
+  const sourcesRoot = path.join(worktreeRoot, "sources");
+  await ensureDirectoryNotSymlink(sourcesRoot, `Sources directory not found: ${sourcesRoot}`);
+
+  const sourceRoot = sourceDirectory(worktreeRoot, documentId);
+  await ensureDirectoryNotSymlink(sourceRoot, `Source directory not found: ${documentId}`);
+  const realSourceRoot = await realpath(sourceRoot);
+  await ensureParentDirectoriesNotSymlink(sourceRoot, fileName);
+  const filePath = path.join(sourceRoot, fileName);
+
+  try {
+    const stat = await lstat(filePath);
+
+    if (stat.isSymbolicLink()) {
+      throw new WikiError("invalid_path", `Source file is a symlink: ${filePath}`);
+    }
+
+    if (!stat.isFile()) {
+      throw new WikiError("invalid_path", `Source path is not a file: ${filePath}`);
+    }
+  } catch (error) {
+    if (isEnoent(error)) {
+      if (optional) {
+        return undefined;
+      }
+
+      throw new WikiError("document_not_found", `Source file not found: ${filePath}`);
+    }
+
+    throw error;
+  }
+
+  const realTarget = await realpath(filePath);
+  ensureInsideRealSourceRoot(realSourceRoot, realTarget, filePath);
+  return filePath;
+}
+
+async function ensureParentDirectoriesNotSymlink(sourceRoot: string, relativeFilePath: string): Promise<void> {
+  const parentSegments = path.dirname(relativeFilePath).split(path.sep).filter(segment => segment.length > 0 && segment !== ".");
+  let currentPath = sourceRoot;
+
+  for (const segment of parentSegments) {
+    currentPath = path.join(currentPath, segment);
+    await ensureDirectoryNotSymlink(currentPath, `Source parent directory not found: ${currentPath}`);
+  }
+}
+
+async function ensureDirectoryNotSymlink(directoryPath: string, missingMessage: string): Promise<void> {
+  let stat;
+
+  try {
+    stat = await lstat(directoryPath);
+  } catch (error) {
+    if (isEnoent(error)) {
+      throw new WikiError("document_not_found", missingMessage);
+    }
+
+    throw error;
+  }
+
+  if (stat.isSymbolicLink()) {
+    throw new WikiError("invalid_path", `Source directory is a symlink: ${directoryPath}`);
+  }
+
+  if (!stat.isDirectory()) {
+    throw new WikiError("invalid_path", `Source path is not a directory: ${directoryPath}`);
+  }
+}
+
+async function ensureConfiguredDirectoryNotSymlink(directoryPath: string, label: string): Promise<void> {
+  let stat;
+
+  try {
+    stat = await lstat(directoryPath);
+  } catch (error) {
+    if (isEnoent(error)) {
+      throw new WikiError("invalid_path", `${label} not found: ${directoryPath}`);
+    }
+
+    throw error;
+  }
+
+  if (stat.isSymbolicLink()) {
+    throw new WikiError("invalid_path", `${label} is a symlink: ${directoryPath}`);
+  }
+
+  if (!stat.isDirectory()) {
+    throw new WikiError("invalid_path", `${label} is not a directory: ${directoryPath}`);
+  }
+}
+
+function ensureInsideRealSourceRoot(realSourceRoot: string, target: string, originalPath: string): void {
+  ensureInsideRealRoot(realSourceRoot, target, originalPath, "Source path escapes source directory");
+}
+
+function ensureStrictDescendant(realRoot: string, target: string, originalPath: string, message: string): void {
+  if (!target.startsWith(`${realRoot}${path.sep}`)) {
+    throw new WikiError("invalid_path", `${message}: ${originalPath}`);
+  }
+}
+
+function ensureInsideRealRoot(realRoot: string, target: string, originalPath: string, message: string): void {
+  if (!target.startsWith(`${realRoot}${path.sep}`) && target !== realRoot) {
+    throw new WikiError("invalid_path", `${message}: ${originalPath}`);
+  }
+}
+
+function parseMetadataJson(raw: string, documentId: string): SourceMetadata {
+  try {
+    const metadata = parseSourceMetadata(JSON.parse(raw));
+    if (metadata.documentId !== documentId) {
+      throw new WikiError("validation_failed", `Source metadata document id mismatch: ${documentId}`, {
+        expected: documentId,
+        actual: metadata.documentId
+      });
+    }
+
+    if (metadata.rawMarkdownPath !== `sources/${documentId}/raw.md`) {
+      throw new WikiError("validation_failed", `Source metadata raw markdown path mismatch: ${documentId}`, {
+        expected: `sources/${documentId}/raw.md`,
+        actual: metadata.rawMarkdownPath
+      });
+    }
+
+    return metadata;
+  } catch (error) {
+    if (error instanceof SyntaxError) {
+      throw new WikiError("validation_failed", `Source metadata is malformed JSON: ${documentId}`, {
+        cause: error.message
+      });
+    }
+
+    throw error;
+  }
+}
+
+function toSourceListItem(metadata: SourceMetadata): SourceListResult["sources"][number] {
+  if (metadata.sourceType === "web") {
+    return {
+      document_id: metadata.documentId,
+      source_type: metadata.sourceType,
+      original_file_name: metadata.title ?? metadata.finalUrl,
+      version: metadata.version,
+      parser: metadata.fetcher.name,
+      raw_markdown_path: metadata.rawMarkdownPath,
+      created_at: metadata.createdAt,
+      updated_at: metadata.updatedAt
+    };
+  }
+
+  return {
+    document_id: metadata.documentId,
+    source_type: metadata.sourceType,
+    original_file_name: metadata.originalFileName,
+    version: metadata.version,
+    parser: metadata.parser.name,
+    raw_markdown_path: metadata.rawMarkdownPath,
+    created_at: metadata.createdAt,
+    updated_at: metadata.updatedAt
+  };
+}
+
+function toSourceError(error: unknown, fallbackCode: "document_not_found"): WikiError {
+  if (error instanceof WikiError) {
+    return error;
+  }
+
+  if (error instanceof Error) {
+    return new WikiError(fallbackCode, error.message);
+  }
+
+  return new WikiError(fallbackCode, String(error));
+}
+
+function isEnoent(error: unknown): boolean {
+  return Boolean(error && typeof error === "object" && "code" in error && error.code === "ENOENT");
+}
+
+function isValidBareSessionRef(ref: string): boolean {
+  try {
+    assertValidSessionId(ref);
+    return true;
+  } catch {
+    return false;
+  }
+}

package/packages/core/src/sources/source-types.ts

@@ -0,0 +1,77 @@
+export type SourceBaseMetadata = {
+  documentId: string;
+  version: number;
+  sourceType: string;
+  createdAt: string;
+  updatedAt: string;
+  status: string;
+  createdBy?: string;
+};
+
+export type SourceParserMetadata = {
+  name: string;
+  version: string;
+};
+
+export type DocumentSourceMetadata = SourceBaseMetadata & {
+  sourceType: "pdf" | "markdown" | "text";
+  originalFileName: string;
+  mimeType: string;
+  blobSha256: string;
+  rawMarkdownPath: string;
+  parser: SourceParserMetadata;
+  status: "parsed";
+};
+
+export type WebFetcherMetadata = {
+  name: "static-fetch";
+  version: string;
+};
+
+export type WebSourceMetadata = SourceBaseMetadata & {
+  sourceType: "web";
+  url: string;
+  finalUrl: string;
+  title?: string;
+  contentType: string;
+  statusCode: number;
+  htmlBlobSha256: string;
+  rawMarkdownPath: string;
+  fetcher: WebFetcherMetadata;
+  fetchMetadata: {
+    htmlBytes: number;
+    markdownBytes: number;
+  };
+  status: "fetched";
+};
+
+export type SourceMetadata = DocumentSourceMetadata | WebSourceMetadata;
+
+export type SourceListInput = {
+  ref?: "stable" | string;
+};
+
+export type SourceReadInput = {
+  document_id: string;
+  ref?: "stable" | string;
+};
+
+export type SourceListResult = {
+  sources: Array<{
+    document_id: string;
+    source_type: string;
+    original_file_name: string;
+    version: number;
+    parser: string;
+    raw_markdown_path: string;
+    created_at: string;
+    updated_at: string;
+  }>;
+};
+
+export type SourceReadResult = {
+  document_id: string;
+  ref: string;
+  metadata: Record<string, unknown>;
+  raw_markdown: string;
+};

package/packages/core/src/sources/source-validator.ts

@@ -0,0 +1,132 @@
+import { WikiError } from "../errors.js";
+import { assertValidSourceId } from "./source-id.js";
+import type { SourceMetadata, SourceParserMetadata, WebFetcherMetadata } from "./source-types.js";
+
+const sha256Pattern = /^[a-fA-F0-9]{64}$/;
+
+export function parseSourceMetadata(value: unknown): SourceMetadata {
+  if (!value || typeof value !== "object") {
+    throw invalidMetadata("Source metadata must be an object");
+  }
+
+  const metadata = value as Record<string, unknown>;
+
+  if (
+    !isString(metadata.documentId) ||
+    !isPositiveInteger(metadata.version) ||
+    !isString(metadata.sourceType) ||
+    !isIsoishString(metadata.createdAt) ||
+    !isIsoishString(metadata.updatedAt) ||
+    !isString(metadata.status) ||
+    !isOptionalString(metadata.createdBy)
+  ) {
+    throw invalidMetadata("Source metadata common fields are invalid", { fields: ["documentId", "version", "sourceType"] });
+  }
+
+  assertValidSourceId(metadata.documentId);
+
+  if (!isSupportedSourceType(metadata.sourceType)) {
+    throw invalidMetadata("Unsupported source metadata type", { sourceType: metadata.sourceType });
+  }
+
+  if (metadata.sourceType === "web") {
+    if (
+      !isNonEmptyString(metadata.url) ||
+      !isNonEmptyString(metadata.finalUrl) ||
+      !isOptionalString(metadata.title) ||
+      !isNonEmptyString(metadata.contentType) ||
+      !isPositiveInteger(metadata.statusCode) ||
+      !isString(metadata.htmlBlobSha256) ||
+      !sha256Pattern.test(metadata.htmlBlobSha256) ||
+      !isString(metadata.rawMarkdownPath) ||
+      !isFetcherMetadata(metadata.fetcher) ||
+      !isWebFetchMetadata(metadata.fetchMetadata) ||
+      metadata.status !== "fetched"
+    ) {
+      throw invalidMetadata("Source metadata web fields are invalid", { documentId: metadata.documentId });
+    }
+
+    return metadata as SourceMetadata;
+  }
+
+  if (
+    !isNonEmptyString(metadata.originalFileName) ||
+    !isSupportedSourceMime(metadata.sourceType, metadata.mimeType) ||
+    !isString(metadata.blobSha256) ||
+    !sha256Pattern.test(metadata.blobSha256) ||
+    !isString(metadata.rawMarkdownPath) ||
+    !isParserMetadata(metadata.parser) ||
+    metadata.status !== "parsed"
+  ) {
+    throw invalidMetadata("Source metadata document fields are invalid", { documentId: metadata.documentId });
+  }
+
+  return metadata as SourceMetadata;
+}
+
+function isSupportedSourceType(value: unknown): value is SourceMetadata["sourceType"] {
+  return value === "pdf" || value === "markdown" || value === "text" || value === "web";
+}
+
+function isSupportedSourceMime(sourceType: unknown, mimeType: unknown): boolean {
+  return (
+    (sourceType === "pdf" && mimeType === "application/pdf") ||
+    (sourceType === "markdown" && (mimeType === "text/markdown" || mimeType === "text/x-markdown")) ||
+    (sourceType === "text" && mimeType === "text/plain")
+  );
+}
+
+function invalidMetadata(message: string, details: Record<string, unknown> = {}): WikiError {
+  return new WikiError("validation_failed", message, details);
+}
+
+function isParserMetadata(value: unknown): value is SourceParserMetadata {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+
+  const parser = value as Partial<SourceParserMetadata>;
+  return isNonEmptyString(parser.name) && isNonEmptyString(parser.version);
+}
+
+function isFetcherMetadata(value: unknown): value is WebFetcherMetadata {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+
+  const fetcher = value as { name?: unknown; version?: unknown };
+  return fetcher.name === "static-fetch" && isNonEmptyString(fetcher.version);
+}
+
+function isWebFetchMetadata(value: unknown): value is { htmlBytes: number; markdownBytes: number } {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+
+  const metadata = value as { htmlBytes?: unknown; markdownBytes?: unknown };
+  return isNonNegativeInteger(metadata.htmlBytes) && isNonNegativeInteger(metadata.markdownBytes);
+}
+
+function isString(value: unknown): value is string {
+  return typeof value === "string";
+}
+
+function isNonEmptyString(value: unknown): value is string {
+  return typeof value === "string" && value.trim().length > 0;
+}
+
+function isOptionalString(value: unknown): value is string | undefined {
+  return value === undefined || typeof value === "string";
+}
+
+function isPositiveInteger(value: unknown): value is number {
+  return typeof value === "number" && Number.isInteger(value) && value > 0;
+}
+
+function isNonNegativeInteger(value: unknown): value is number {
+  return typeof value === "number" && Number.isInteger(value) && value >= 0;
+}
+
+function isIsoishString(value: unknown): value is string {
+  return typeof value === "string" && !Number.isNaN(Date.parse(value));
+}