wiki-graph-sdk 0.7.0

package/dist/worker.js

@@ -0,0 +1,715 @@
+import { fileURLToPath as __WIKIGRAPH_FILE_URL_TO_PATH__ } from "url";
+import { resolve as __WIKIGRAPH_RESOLVE__ } from "path";
+globalThis.__WIKIGRAPH_DATA_DIR__ ??= __WIKIGRAPH_RESOLVE__(__WIKIGRAPH_FILE_URL_TO_PATH__(new URL("./data", import.meta.url)));
+
+// src/worker.ts
+import {
+  runBuildJobWorker as runCoreBuildJobWorker
+} from "wiki-graph-core/worker";
+
+// src/node-platform.ts
+import * as asyncHooks from "async_hooks";
+import * as crypto from "crypto";
+import * as fs from "fs";
+import * as fsPromises from "fs/promises";
+import * as os from "os";
+import * as pathModule from "path";
+import * as process from "process";
+import { Readable, Writable } from "stream";
+import { pipeline } from "stream/promises";
+import * as sqlite3 from "sqlite3";
+import * as yauzl from "yauzl";
+import * as yazl from "yazl";
+import { Environment, Loader } from "nunjucks";
+import {
+  getWikiGraphStorage,
+  installWikiGraphPlatform,
+  installWikiGraphStorage,
+  withWikiGraphStorage,
+  normalizeTemplateName
+} from "wiki-graph-core/platform";
+var nodeProcess = process.default ?? process;
+var nodeSqlite3 = sqlite3.default ?? sqlite3;
+var NODE_RESOURCE_KIND = /* @__PURE__ */ Symbol.for("wiki-graph-sdk.node-resource.v1");
+var NodeFile = class {
+  constructor(path, name = pathModule.basename(path)) {
+    this.path = path;
+    this.identity = encodeNodeResourceIdentity("file", path);
+    this.name = name;
+  }
+  path;
+  kind = "file";
+  [NODE_RESOURCE_KIND] = "file";
+  identity;
+  name;
+  async openReader() {
+    const handle = await fsPromises.open(this.path, "r");
+    let closed = false;
+    try {
+      const stats = await handle.stat();
+      if (!stats.isFile())
+        throw new Error("File reader requires a regular file");
+      const { size } = stats;
+      return {
+        size,
+        close: async () => {
+          if (!closed) await handle.close();
+          closed = true;
+        },
+        read: async (offset, length) => {
+          if (closed) throw new Error("Cannot read from a closed FileReader");
+          assertFileRange(offset, length, size);
+          const buffer = Buffer.allocUnsafe(length);
+          let bytesRead = 0;
+          while (bytesRead < length) {
+            const result = await handle.read(
+              buffer,
+              bytesRead,
+              length - bytesRead,
+              offset + bytesRead
+            );
+            if (result.bytesRead === 0) {
+              throw new Error("File changed while reading its byte range");
+            }
+            bytesRead += result.bytesRead;
+          }
+          return new Uint8Array(buffer.buffer, buffer.byteOffset, bytesRead);
+        }
+      };
+    } catch (error) {
+      await handle.close();
+      throw error;
+    }
+  }
+  async getLastModified() {
+    try {
+      return (await fsPromises.stat(this.path)).mtimeMs;
+    } catch (error) {
+      if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
+        return void 0;
+      }
+      throw error;
+    }
+  }
+  async openWriter() {
+    await fsPromises.mkdir(pathModule.dirname(this.path), { recursive: true });
+    const temporary = `${this.path}.tmp-${crypto.randomUUID()}`;
+    const handle = await fsPromises.open(temporary, "wx");
+    let closed = false;
+    return {
+      write: async (data) => {
+        if (closed) throw new Error("Cannot write to a closed FileWriter");
+        await writeNodeFileHandle(handle, data);
+      },
+      writeAt: async (offset, data) => {
+        if (closed) throw new Error("Cannot write to a closed FileWriter");
+        assertFileOffset(offset);
+        await writeNodeFileHandle(handle, data, offset);
+      },
+      commit: async () => {
+        if (!closed) {
+          await handle.close();
+          await fsPromises.rename(temporary, this.path);
+        }
+        closed = true;
+      },
+      abort: async () => {
+        if (!closed) {
+          await handle.close();
+          await fsPromises.rm(temporary, { force: true });
+        }
+        closed = true;
+      }
+    };
+  }
+};
+async function writeNodeFileHandle(handle, data, position) {
+  const buffer = Buffer.from(data);
+  let written = 0;
+  while (written < buffer.byteLength) {
+    const result = await handle.write(
+      buffer,
+      written,
+      buffer.byteLength - written,
+      position === void 0 ? null : position + written
+    );
+    if (result.bytesWritten === 0) {
+      throw new Error("File writer made no progress");
+    }
+    written += result.bytesWritten;
+  }
+}
+function assertFileRange(offset, length, size) {
+  assertFileOffset(offset);
+  if (!Number.isSafeInteger(length) || length < 0) {
+    throw new RangeError(
+      "File read length must be a non-negative safe integer"
+    );
+  }
+  if (offset > size || length > size - offset) {
+    throw new RangeError("File read range exceeds the file size");
+  }
+}
+function assertFileOffset(offset) {
+  if (!Number.isSafeInteger(offset) || offset < 0) {
+    throw new RangeError("File offset must be a non-negative safe integer");
+  }
+}
+var NodeDirectory = class _NodeDirectory {
+  constructor(path) {
+    this.path = path;
+    this.identity = encodeNodeResourceIdentity("directory", path);
+    this.name = pathModule.basename(path);
+  }
+  path;
+  kind = "directory";
+  [NODE_RESOURCE_KIND] = "directory";
+  identity;
+  name;
+  async getFile(name) {
+    assertChildName(name);
+    const file = new NodeFile(pathModule.join(this.path, name));
+    try {
+      return (await fsPromises.stat(file.path)).isFile() ? file : void 0;
+    } catch {
+      return void 0;
+    }
+  }
+  async getLastModified() {
+    try {
+      return (await fsPromises.stat(this.path)).mtimeMs;
+    } catch (error) {
+      if (typeof error === "object" && error !== null && "code" in error && error.code === "ENOENT") {
+        return void 0;
+      }
+      throw error;
+    }
+  }
+  async getDirectory(name) {
+    assertChildName(name);
+    const directory = new _NodeDirectory(pathModule.join(this.path, name));
+    try {
+      return (await fsPromises.stat(directory.path)).isDirectory() ? directory : void 0;
+    } catch {
+      return void 0;
+    }
+  }
+  async list() {
+    const entries = await fsPromises.readdir(this.path, {
+      withFileTypes: true
+    });
+    return entries.map(
+      (entry) => entry.isDirectory() ? new _NodeDirectory(pathModule.join(this.path, entry.name)) : new NodeFile(pathModule.join(this.path, entry.name), entry.name)
+    );
+  }
+  async createFile(name) {
+    assertChildName(name);
+    await fsPromises.mkdir(this.path, { recursive: true });
+    const file = new NodeFile(pathModule.join(this.path, name));
+    await fsPromises.writeFile(file.path, "", { flag: "wx" });
+    return file;
+  }
+  async createDirectory(name) {
+    assertChildName(name);
+    await fsPromises.mkdir(this.path, { recursive: true });
+    const directory = new _NodeDirectory(pathModule.join(this.path, name));
+    await fsPromises.mkdir(directory.path);
+    return directory;
+  }
+  async remove(name, options) {
+    assertChildName(name);
+    await fsPromises.rm(pathModule.join(this.path, name), {
+      force: true,
+      recursive: options?.recursive === true
+    });
+  }
+};
+function encodeNodeResourceIdentity(kind, path) {
+  return `node-${kind}:${Buffer.from(pathModule.resolve(path), "utf8").toString("base64url")}`;
+}
+function decodeNodeResourceIdentity(identity, kind) {
+  const prefix = `node-${kind}:`;
+  if (!identity.startsWith(prefix)) {
+    return pathModule.isAbsolute(identity) ? identity : void 0;
+  }
+  try {
+    return Buffer.from(identity.slice(prefix.length), "base64url").toString(
+      "utf8"
+    );
+  } catch {
+    return void 0;
+  }
+}
+function assertChildName(name) {
+  if (name.length === 0 || name === "." || name === ".." || pathModule.isAbsolute(name) || name.includes("/") || name.includes("\\")) {
+    throw new TypeError(`Directory child must be a relative name: ${name}`);
+  }
+}
+var NodeDatabaseConnection = class {
+  constructor(database) {
+    this.database = database;
+  }
+  database;
+  async close() {
+    await new Promise((resolve2, reject) => {
+      this.database.close((error) => {
+        if (error) reject(error);
+        else resolve2();
+      });
+    });
+  }
+  async execute(sql) {
+    await new Promise((resolve2, reject) => {
+      this.database.exec(sql, (error) => {
+        if (error) reject(error);
+        else resolve2();
+      });
+    });
+  }
+  async queryAll(sql, params = []) {
+    return await new Promise((resolve2, reject) => {
+      this.database.all(sql, [...params], (error, rows) => {
+        if (error) reject(error);
+        else resolve2(rows);
+      });
+    });
+  }
+  async queryOne(sql, params = []) {
+    return await new Promise((resolve2, reject) => {
+      this.database.get(sql, [...params], (error, row) => {
+        if (error) reject(error);
+        else resolve2(row);
+      });
+    });
+  }
+  async run(sql, params = []) {
+    await new Promise((resolve2, reject) => {
+      this.database.run(sql, [...params], (error) => {
+        if (error) reject(error);
+        else resolve2();
+      });
+    });
+  }
+};
+async function openNodeDatabase(file, options) {
+  const flags = (options.mode === "readonly" ? nodeSqlite3.OPEN_READONLY : nodeSqlite3.OPEN_READWRITE | (options.create ? nodeSqlite3.OPEN_CREATE : 0)) | nodeSqlite3.OPEN_FULLMUTEX;
+  return new NodeDatabaseConnection(await openNativeNodeDatabase(file, flags));
+}
+async function openNativeNodeDatabase(file, flags) {
+  const path = getBrandedNodeResourcePath(file, "file");
+  if (path === void 0) {
+    throw new TypeError("The Node database adapter requires a NodeFile");
+  }
+  return await new Promise((resolve2, reject) => {
+    const database = new nodeSqlite3.Database(path, flags, (error) => {
+      if (error) reject(error);
+      else resolve2(database);
+    });
+  });
+}
+async function openNodeZip(file) {
+  const path = getBrandedNodeResourcePath(file, "file");
+  if (path === void 0) {
+    throw new TypeError("The Node ZIP adapter requires a NodeFile");
+  }
+  const zipFile = await new Promise((resolve2, reject) => {
+    yauzl.open(
+      path,
+      { autoClose: false, lazyEntries: true },
+      (error, opened) => {
+        if (error || opened === void 0)
+          reject(error ?? new Error("Cannot open ZIP"));
+        else resolve2(opened);
+      }
+    );
+  });
+  const entries = await new Promise(
+    (resolve2, reject) => {
+      const discovered = /* @__PURE__ */ new Map();
+      const rejectAndClose = (error) => {
+        zipFile.close();
+        reject(
+          error instanceof Error ? error : new Error("Cannot scan ZIP central directory")
+        );
+      };
+      const resolveEntries = () => {
+        zipFile.off("error", rejectAndClose);
+        resolve2(discovered);
+      };
+      zipFile.on("entry", (entry) => {
+        if (!entry.fileName.endsWith("/")) {
+          discovered.set(entry.fileName, entry);
+        }
+        zipFile.readEntry();
+      });
+      zipFile.once("error", rejectAndClose);
+      zipFile.once("end", resolveEntries);
+      zipFile.readEntry();
+    }
+  );
+  let closed = false;
+  return {
+    close: () => {
+      if (!closed) zipFile.close();
+      closed = true;
+      return Promise.resolve();
+    },
+    copyEntry: async (name, target) => {
+      if (closed) throw new Error("Cannot read a closed ZIP archive");
+      const entry = entries.get(name);
+      if (entry === void 0) return false;
+      await copyNodeZipEntry(zipFile, entry, target);
+      return true;
+    },
+    getEntrySize: (name) => {
+      if (closed) throw new Error("Cannot read a closed ZIP archive");
+      return Promise.resolve(entries.get(name)?.uncompressedSize);
+    },
+    listEntries: () => Promise.resolve([...entries.keys()]),
+    readEntryRange: async (name, offset, length) => {
+      if (closed) throw new Error("Cannot read a closed ZIP archive");
+      const entry = entries.get(name);
+      if (entry === void 0) return void 0;
+      return await readNodeZipEntryRange(zipFile, entry, offset, length);
+    },
+    readEntry: async (name) => {
+      if (closed) throw new Error("Cannot read a closed ZIP archive");
+      const entry = entries.get(name);
+      if (entry === void 0) return void 0;
+      return await readNodeZipEntry(zipFile, entry);
+    }
+  };
+}
+async function copyNodeZipEntry(zipFile, entry, target) {
+  const writer = await target.openWriter();
+  try {
+    const stream = await openNodeZipEntryStream(zipFile, entry);
+    await writeNodeStream(stream, writer);
+    await writer.commit();
+  } catch (error) {
+    await writer.abort().catch(() => void 0);
+    throw error;
+  }
+}
+async function readNodeZipEntry(zipFile, entry) {
+  return await collectNodeStream(await openNodeZipEntryStream(zipFile, entry));
+}
+async function readNodeZipEntryRange(zipFile, entry, offset, length) {
+  assertFileRange(offset, length, entry.uncompressedSize);
+  if (length === 0) return new Uint8Array();
+  if (entry.compressionMethod !== 0) {
+    const content2 = await collectDecodedNodeStreamRange(
+      await openNodeZipEntryStream(zipFile, entry),
+      offset,
+      length
+    );
+    return new Uint8Array(
+      content2.buffer,
+      content2.byteOffset,
+      content2.byteLength
+    );
+  }
+  const content = await collectNodeStream(
+    await openNodeZipEntryStream(zipFile, entry, {
+      decompress: null,
+      decrypt: null,
+      end: offset + length,
+      start: offset
+    })
+  );
+  return new Uint8Array(content.buffer, content.byteOffset, content.byteLength);
+}
+async function collectDecodedNodeStreamRange(input, offset, length) {
+  const chunks = [];
+  const end = offset + length;
+  let position = 0;
+  let collected = 0;
+  for await (const value of input) {
+    const chunk = Buffer.isBuffer(value) ? value : Buffer.from(value);
+    const chunkEnd = position + chunk.byteLength;
+    if (chunkEnd > offset && position < end) {
+      const selected = chunk.subarray(
+        Math.max(0, offset - position),
+        Math.min(chunk.byteLength, end - position)
+      );
+      chunks.push(selected);
+      collected += selected.byteLength;
+    }
+    position = chunkEnd;
+    if (position >= end) break;
+  }
+  if (collected !== length) {
+    throw new Error("ZIP entry changed while reading its byte range");
+  }
+  return Buffer.concat(chunks, collected);
+}
+async function openNodeZipEntryStream(zipFile, entry, options) {
+  return await new Promise((resolve2, reject) => {
+    const callback = (error, opened) => {
+      if (error || opened === void 0) {
+        reject(error ?? new Error(`Cannot read ZIP entry: ${entry.fileName}`));
+      } else {
+        resolve2(opened);
+      }
+    };
+    if (options === void 0) zipFile.openReadStream(entry, callback);
+    else zipFile.openReadStream(entry, options, callback);
+  });
+}
+async function writeNodeZip(file, entries) {
+  const zipFile = new yazl.ZipFile();
+  const writer = await file.openWriter();
+  const outputStream = zipFile.outputStream;
+  const outputDone = writeNodeStream(outputStream, writer);
+  const entryState = { cancelled: false, complete: false };
+  let iterator;
+  try {
+    iterator = getNodeZipEntryIterator(entries);
+    const entriesDone = addNodeZipEntries(zipFile, iterator, entryState);
+    await Promise.race([entriesDone, outputDone]);
+    if (!entryState.complete) {
+      throw new Error("ZIP output ended before entry production completed");
+    }
+    zipFile.end();
+    await outputDone;
+    await writer.commit();
+  } catch (error) {
+    entryState.cancelled = true;
+    if (!entryState.complete && iterator !== void 0) {
+      closeNodeZipEntryIterator(iterator);
+    }
+    if (!outputStream.destroyed) {
+      outputStream.destroy(
+        error instanceof Error ? error : new Error("Cannot write ZIP archive")
+      );
+    }
+    await outputDone.catch(() => void 0);
+    await writer.abort().catch(() => void 0);
+    throw error;
+  }
+}
+async function addNodeZipEntries(zipFile, iterator, state) {
+  while (!state.cancelled) {
+    const result = await iterator.next();
+    if (state.cancelled) return;
+    if (result.done) {
+      state.complete = true;
+      return;
+    }
+    if ("file" in result.value) {
+      zipFile.addReadStream(
+        createNodeFileReadStream(result.value.file),
+        result.value.name,
+        { compress: false }
+      );
+    } else if ("data" in result.value) {
+      zipFile.addBuffer(Buffer.from(result.value.data), result.value.name, {
+        compress: false
+      });
+    } else {
+      zipFile.addReadStream(
+        createNodeRangeReadStream(result.value),
+        result.value.name,
+        { compress: false, size: result.value.size }
+      );
+    }
+  }
+}
+function createNodeFileReadStream(file) {
+  return Readable.from(
+    (async function* () {
+      const reader = await file.openReader();
+      try {
+        for (let offset = 0; offset < reader.size; ) {
+          const chunk = await reader.read(
+            offset,
+            Math.min(64 * 1024, reader.size - offset)
+          );
+          offset += chunk.byteLength;
+          yield chunk;
+        }
+      } finally {
+        await reader.close();
+      }
+    })()
+  );
+}
+function getBrandedNodeResourcePath(resource, kind) {
+  if (typeof resource !== "object" || resource === null) return void 0;
+  const candidate = resource;
+  return candidate[NODE_RESOURCE_KIND] === kind && typeof candidate.path === "string" ? candidate.path : void 0;
+}
+function createNodeRangeReadStream(entry) {
+  return Readable.from(
+    (async function* () {
+      for (let offset = 0; offset < entry.size; ) {
+        const chunk = await entry.read(
+          offset,
+          Math.min(64 * 1024, entry.size - offset)
+        );
+        offset += chunk.byteLength;
+        yield chunk;
+      }
+    })()
+  );
+}
+function getNodeZipEntryIterator(entries) {
+  const asyncIterator = entries[Symbol.asyncIterator];
+  return asyncIterator === void 0 ? entries[Symbol.iterator]() : asyncIterator.call(entries);
+}
+function closeNodeZipEntryIterator(iterator) {
+  if (iterator.return === void 0) return;
+  try {
+    void Promise.resolve(iterator.return()).catch(() => void 0);
+  } catch {
+  }
+}
+async function writeNodeStream(input, writer) {
+  await pipeline(
+    input,
+    new Writable({
+      write: (chunk, _encoding, callback) => {
+        void writer.write(chunk).then(
+          () => callback(),
+          (error) => callback(
+            error instanceof Error ? error : new Error("Cannot stream ZIP archive")
+          )
+        );
+      }
+    })
+  );
+}
+async function collectNodeStream(input) {
+  return await new Promise((resolve2, reject) => {
+    const chunks = [];
+    input.on("data", (chunk) => chunks.push(chunk));
+    input.once("error", reject);
+    input.once("end", () => resolve2(Buffer.concat(chunks)));
+  });
+}
+var nodeWikiGraphPlatform = {
+  asyncContext: {
+    create: () => new asyncHooks.AsyncLocalStorage()
+  },
+  database: { open: openNodeDatabase },
+  lifecycle: {
+    instanceId: `node-process:${nodeProcess.pid}`,
+    isInstanceAlive: (instanceId) => {
+      const match = /^node-process:(\d+)$/u.exec(instanceId);
+      if (match === null) return Promise.resolve(void 0);
+      try {
+        nodeProcess.kill(Number(match[1]), 0);
+        return Promise.resolve(true);
+      } catch (error) {
+        if (typeof error === "object" && error !== null && "code" in error && error.code === "ESRCH") {
+          return Promise.resolve(false);
+        }
+        return Promise.resolve(true);
+      }
+    }
+  },
+  resources: {
+    getDirectory: (identity) => {
+      const path = decodeNodeResourceIdentity(identity, "directory");
+      return Promise.resolve(
+        path === void 0 ? void 0 : new NodeDirectory(path)
+      );
+    },
+    getFile: (identity) => {
+      const path = decodeNodeResourceIdentity(identity, "file");
+      return Promise.resolve(
+        path === void 0 ? void 0 : new NodeFile(path)
+      );
+    },
+    resolveLegacyDirectory: (reference) => {
+      const path = decodeNodeResourceIdentity(reference, "directory");
+      return Promise.resolve(
+        path === void 0 ? void 0 : new NodeDirectory(path)
+      );
+    }
+  },
+  templates: {
+    createEnvironment: (options) => new Environment(new NodeTemplateLoader(), options)
+  },
+  zip: {
+    open: openNodeZip,
+    write: writeNodeZip
+  }
+};
+var NodeTemplateLoader = class extends Loader {
+  #dataDirectory;
+  constructor(dataDirectory) {
+    super();
+    this.#dataDirectory = dataDirectory;
+  }
+  getSource(templateName) {
+    const name = normalizeTemplateName(templateName);
+    const filePath = pathModule.join(
+      this.#dataDirectory ?? resolveNodeDataDirectory(),
+      name
+    );
+    return {
+      noCache: false,
+      path: name,
+      src: fs.readFileSync(filePath, "utf8")
+    };
+  }
+};
+function resolveNodeDataDirectory() {
+  const injected = globalThis.__WIKIGRAPH_DATA_DIR__;
+  if (typeof injected === "string" && injected !== "") return injected;
+  let directory = nodeProcess.cwd();
+  while (true) {
+    for (const candidate of [
+      pathModule.join(directory, "data"),
+      pathModule.join(directory, "packages", "core", "data")
+    ]) {
+      try {
+        if (fs.statSync(candidate).isDirectory()) return candidate;
+      } catch {
+      }
+    }
+    const parent = pathModule.dirname(directory);
+    if (parent === directory)
+      throw new Error("Could not locate data directory");
+    directory = parent;
+  }
+}
+function createNodeWikiGraphStorage(stateRoot = pathModule.join(os.homedir(), ".wikigraph")) {
+  return {
+    library: new NodeDirectory(stateRoot),
+    documentStore: new NodeDirectory(pathModule.join(stateRoot, "documents"))
+  };
+}
+function installNodeWikiGraphPlatform(stateRoot) {
+  installWikiGraphPlatform(nodeWikiGraphPlatform);
+  installWikiGraphStorage(createNodeWikiGraphStorage(stateRoot));
+}
+async function withNodeWikiGraphStorage(stateRoot, operation) {
+  return await withWikiGraphStorage(
+    stateRoot === void 0 ? void 0 : createNodeWikiGraphStorage(stateRoot),
+    operation
+  );
+}
+
+// src/worker.ts
+import {
+  applyChapterJobArtifactFile,
+  createLocalChapterJobFileExecutor,
+  writeChapterJobInputFile
+} from "wiki-graph-core/worker";
+async function runBuildJobWorker(options) {
+  installNodeWikiGraphPlatform();
+  const { stateDir, ...coreOptions } = options;
+  await withNodeWikiGraphStorage(
+    stateDir,
+    async () => await runCoreBuildJobWorker(coreOptions)
+  );
+}
+export {
+  applyChapterJobArtifactFile,
+  createLocalChapterJobFileExecutor,
+  runBuildJobWorker,
+  writeChapterJobInputFile
+};
+//# sourceMappingURL=worker.js.map