plugin-document-parser 1.0.1

package/dist/server/services/external-ocr-client.js

@@ -0,0 +1,189 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var external_ocr_client_exports = {};
+__export(external_ocr_client_exports, {
+  callExternalOcr: () => callExternalOcr,
+  testOcrProviderConnection: () => testOcrProviderConnection
+});
+module.exports = __toCommonJS(external_ocr_client_exports);
+var import_axios = __toESM(require("axios"));
+var import_form_data = __toESM(require("form-data"));
+function setByPath(obj, dotPath, value) {
+  const parts = dotPath.split(".");
+  let current = obj;
+  for (let i = 0; i < parts.length - 1; i++) {
+    if (current[parts[i]] == null || typeof current[parts[i]] !== "object") {
+      current[parts[i]] = {};
+    }
+    current = current[parts[i]];
+  }
+  current[parts[parts.length - 1]] = value;
+}
+function getByPath(obj, dotPath) {
+  if (!obj || !dotPath) return void 0;
+  const parts = dotPath.split(".");
+  let current = obj;
+  for (const part of parts) {
+    if (current == null) return void 0;
+    current = current[part];
+  }
+  return current;
+}
+function buildAuthHeaders(config) {
+  const { authType, apiKey, authConfig = {} } = config;
+  const headers = {};
+  switch (authType) {
+    case "bearer":
+      if (apiKey) headers["Authorization"] = `Bearer ${apiKey}`;
+      break;
+    case "api-key-header": {
+      const headerName = authConfig.headerName || "X-Api-Key";
+      if (apiKey) headers[headerName] = apiKey;
+      break;
+    }
+    case "basic": {
+      const { username = "", password = "" } = authConfig;
+      const encoded = Buffer.from(`${username}:${password}`).toString("base64");
+      headers["Authorization"] = `Basic ${encoded}`;
+      break;
+    }
+    case "custom-headers":
+      Object.assign(headers, authConfig.customHeaders ?? {});
+      break;
+    case "none":
+    default:
+      break;
+  }
+  return headers;
+}
+async function callExternalOcr(providerConfig, options) {
+  const {
+    apiEndpoint,
+    requestFormat = "multipart",
+    requestConfig = {},
+    responseTextPath = "text",
+    timeout = 6e4
+  } = providerConfig;
+  const authHeaders = buildAuthHeaders(providerConfig);
+  let axiosConfig = {
+    timeout,
+    headers: { ...authHeaders }
+  };
+  let body;
+  if (requestFormat === "multipart") {
+    const form = new import_form_data.default();
+    const fileFieldName = requestConfig.fileFieldName || "file";
+    form.append(fileFieldName, options.fileBuffer, {
+      filename: options.filename,
+      contentType: options.mimetype
+    });
+    if (requestConfig.filenameFieldName) {
+      form.append(requestConfig.filenameFieldName, options.filename);
+    }
+    if (requestConfig.mimetypeFieldName) {
+      form.append(requestConfig.mimetypeFieldName, options.mimetype);
+    }
+    for (const [k, v] of Object.entries(requestConfig.extraFields ?? {})) {
+      form.append(k, v);
+    }
+    body = form;
+    axiosConfig.headers = {
+      ...axiosConfig.headers,
+      ...form.getHeaders()
+    };
+  } else if (requestFormat === "json-base64") {
+    const base64 = options.fileBuffer.toString("base64");
+    const jsonBody = { ...requestConfig.extraBody ?? {} };
+    setByPath(jsonBody, requestConfig.base64FieldPath || "file", base64);
+    if (requestConfig.filenameFieldPath) {
+      setByPath(jsonBody, requestConfig.filenameFieldPath, options.filename);
+    }
+    if (requestConfig.mimetypeFieldPath) {
+      setByPath(jsonBody, requestConfig.mimetypeFieldPath, options.mimetype);
+    }
+    body = jsonBody;
+    axiosConfig.headers = {
+      ...axiosConfig.headers,
+      "Content-Type": "application/json"
+    };
+  } else if (requestFormat === "url") {
+    if (!options.fileUrl) {
+      throw new Error("[DocumentParser] requestFormat=url but no fileUrl was provided");
+    }
+    const jsonBody = { ...requestConfig.extraBody ?? {} };
+    setByPath(jsonBody, requestConfig.urlFieldPath || "url", options.fileUrl);
+    body = jsonBody;
+    axiosConfig.headers = {
+      ...axiosConfig.headers,
+      "Content-Type": "application/json"
+    };
+  } else {
+    throw new Error(`[DocumentParser] Unknown requestFormat: ${requestFormat}`);
+  }
+  const response = await import_axios.default.post(apiEndpoint, body, axiosConfig);
+  const responseData = response.data;
+  const text = getByPath(responseData, responseTextPath);
+  if (typeof text !== "string") {
+    throw new Error(
+      `[DocumentParser] Could not extract text from response at path "${responseTextPath}". Response: ${JSON.stringify(responseData).slice(0, 300)}`
+    );
+  }
+  return text;
+}
+async function testOcrProviderConnection(providerConfig) {
+  try {
+    const authHeaders = buildAuthHeaders(providerConfig);
+    const response = await import_axios.default.get(providerConfig.apiEndpoint, {
+      headers: authHeaders,
+      timeout: providerConfig.timeout ?? 1e4,
+      // Don't throw on 4xx so we can return the status to the UI
+      validateStatus: () => true
+    });
+    return {
+      ok: response.status < 500,
+      status: response.status,
+      message: response.status < 500 ? void 0 : `Server error: ${response.status}`
+    };
+  } catch (err) {
+    return { ok: false, message: (err == null ? void 0 : err.message) ?? String(err) };
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  callExternalOcr,
+  testOcrProviderConnection
+});

package/dist/server/services/internal-parser-registry.js

@@ -0,0 +1,82 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var internal_parser_registry_exports = {};
+__export(internal_parser_registry_exports, {
+  InternalParserRegistry: () => InternalParserRegistry
+});
+module.exports = __toCommonJS(internal_parser_registry_exports);
+class InternalParserRegistry {
+  handlers = [];
+  /**
+   * Register a new handler.
+   * Pass `{ prepend: true }` to insert at the front so it takes priority
+   * over previously registered handlers (useful for specialised format plugins).
+   */
+  register(handler, options) {
+    if (this.handlers.find((h) => h.name === handler.name)) {
+      throw new Error(`[DocumentParser] Handler "${handler.name}" is already registered.`);
+    }
+    if (options == null ? void 0 : options.prepend) {
+      this.handlers.unshift(handler);
+    } else {
+      this.handlers.push(handler);
+    }
+  }
+  /** Remove a previously registered handler by name */
+  unregister(name) {
+    this.handlers = this.handlers.filter((h) => h.name !== name);
+  }
+  /** Return a copy of the current handler list (for introspection / tests) */
+  list() {
+    return [...this.handlers];
+  }
+  /**
+   * Try handlers in order; return the result of the first one that
+   * `supports` and successfully `parse`s the attachment.
+   *
+   * Returns `{ text: '', handled: false }` when no handler supports the file.
+   */
+  async parse(attachment, ctx) {
+    for (const handler of this.handlers) {
+      if (!handler.supports(attachment)) {
+        continue;
+      }
+      const result = await handler.parse(attachment, ctx);
+      if (result.handled) {
+        return result;
+      }
+    }
+    return { text: "", handled: false };
+  }
+  get size() {
+    return this.handlers.length;
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  InternalParserRegistry
+});

package/dist/server/services/parse-router.js

@@ -0,0 +1,273 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var parse_router_exports = {};
+__export(parse_router_exports, {
+  ParseRouter: () => ParseRouter
+});
+module.exports = __toCommonJS(parse_router_exports);
+var import_os = require("os");
+var import_path = require("path");
+var import_promises = require("fs/promises");
+var import_external_ocr_client = require("./external-ocr-client");
+class ParseRouter {
+  constructor(getSettingsRepo, getProvidersRepo, internalRegistry, getFileBuffer, getDocpixiePlugin = () => null) {
+    this.getSettingsRepo = getSettingsRepo;
+    this.getProvidersRepo = getProvidersRepo;
+    this.internalRegistry = internalRegistry;
+    this.getFileBuffer = getFileBuffer;
+    this.getDocpixiePlugin = getDocpixiePlugin;
+  }
+  // ── Settings cache (invalidated every N ms) ───────────────────────────────
+  cachedSettings = null;
+  settingsCachedAt = 0;
+  CACHE_TTL_MS = 5e3;
+  async getSettings() {
+    const now = Date.now();
+    if (this.cachedSettings && now - this.settingsCachedAt < this.CACHE_TTL_MS) {
+      return this.cachedSettings;
+    }
+    const repo = this.getSettingsRepo();
+    let record = await repo.findOne({});
+    if (!record) {
+      record = await repo.create({
+        values: {
+          mode: "default",
+          fallbackToDefault: true,
+          imagePassThrough: true,
+          includedExtnames: []
+        }
+      });
+    }
+    this.cachedSettings = {
+      mode: record.get("mode") ?? "default",
+      activeProviderId: record.get("activeProviderId") ?? null,
+      fallbackToDefault: record.get("fallbackToDefault") ?? true,
+      imagePassThrough: record.get("imagePassThrough") ?? true,
+      includedExtnames: record.get("includedExtnames") ?? [],
+      useDocpixie: record.get("useDocpixie") ?? false
+    };
+    this.settingsCachedAt = now;
+    return this.cachedSettings;
+  }
+  /** Call after saving settings so the next request reads fresh values */
+  invalidateSettingsCache() {
+    this.cachedSettings = null;
+  }
+  // ── Main entry point ──────────────────────────────────────────────────────
+  async route(ctx, attachment, defaultParser) {
+    var _a;
+    const settings = await this.getSettings();
+    if (settings.imagePassThrough && ((_a = attachment.mimetype) == null ? void 0 : _a.startsWith("image/"))) {
+      return defaultParser();
+    }
+    if (settings.includedExtnames.length > 0) {
+      const ext = resolveExtname(attachment);
+      if (!settings.includedExtnames.includes(ext)) {
+        return defaultParser();
+      }
+    }
+    if (settings.useDocpixie) {
+      const docpixieResult = await this.routeDocpixie(ctx, attachment);
+      if (docpixieResult) return docpixieResult;
+    }
+    switch (settings.mode) {
+      case "default":
+        return defaultParser();
+      case "internal":
+        return this.routeInternal(ctx, attachment, settings, defaultParser);
+      case "external":
+        return this.routeExternal(ctx, attachment, settings, defaultParser);
+      default:
+        return defaultParser();
+    }
+  }
+  // ── DocPixie routing ─────────────────────────────────────────────────────
+  /**
+   * Index the attachment into DocPixie and return a metadata reference block.
+   *
+   * Strategy:
+   *   1. Get the DocPixie plugin instance (returns null if not loaded/active)
+   *   2. Download the file buffer and write to a temp file (works for S3 / local)
+   *   3. Call docpixieService.processDocument() — this:
+   *        a. Creates a DB record immediately → returns documentId fast
+   *        b. Continues extracting pages + summarizing in the background
+   *   4. Build a content block with metadata + LLM instructions to use docpixie:query
+   *   5. Clean up temp file
+   *
+   * Returns null if DocPixie is unavailable or not ready so caller can fall through.
+   */
+  async routeDocpixie(ctx, attachment) {
+    var _a, _b, _c, _d, _e, _f;
+    const docpixiePlugin = this.getDocpixiePlugin();
+    if (!(docpixiePlugin == null ? void 0 : docpixiePlugin.service)) {
+      return null;
+    }
+    const service = docpixiePlugin.service;
+    if (!service.isReady()) {
+      (_b = (_a = ctx.log) == null ? void 0 : _a.warn) == null ? void 0 : _b.call(_a, "[DocumentParser] DocPixie service is not ready (not configured) \u2014 skipping");
+      return null;
+    }
+    const filename = attachment.filename ?? attachment.name ?? "document";
+    const mimetype = attachment.mimetype ?? "application/octet-stream";
+    let tempPath = null;
+    try {
+      const { buffer } = await this.getFileBuffer(ctx, attachment);
+      const ext = resolveExtname(attachment) || ".bin";
+      tempPath = (0, import_path.join)((0, import_os.tmpdir)(), `docparser-${Date.now()}-${Math.random().toString(36).slice(2)}${ext}`);
+      await (0, import_promises.writeFile)(tempPath, buffer);
+      const documentId = await service.processDocument(tempPath, { name: filename });
+      (_d = (_c = ctx.log) == null ? void 0 : _c.info) == null ? void 0 : _d.call(_c, `[DocumentParser] DocPixie indexing started: documentId=${documentId} file="${filename}"`);
+      return docpixieReferenceBlock(documentId, filename, mimetype);
+    } catch (err) {
+      (_f = (_e = ctx.log) == null ? void 0 : _e.warn) == null ? void 0 : _f.call(_e, `[DocumentParser] DocPixie indexing failed for "${filename}": ${err}`);
+      return null;
+    } finally {
+      if (tempPath) {
+        (0, import_promises.unlink)(tempPath).catch(() => {
+        });
+      }
+    }
+  }
+  // ── Internal routing ──────────────────────────────────────────────────────
+  async routeInternal(ctx, attachment, settings, defaultParser) {
+    var _a, _b;
+    try {
+      const result = await this.internalRegistry.parse(attachment, ctx);
+      if (!result.handled) {
+        return settings.fallbackToDefault ? defaultParser() : this.unsupportedResult(attachment);
+      }
+      return textToContentBlock(result.text, attachment);
+    } catch (err) {
+      (_b = (_a = ctx.log) == null ? void 0 : _a.warn) == null ? void 0 : _b.call(_a, `[DocumentParser] internal parse failed for "${attachment.filename}": ${err}`);
+      if (settings.fallbackToDefault) {
+        return defaultParser();
+      }
+      throw err;
+    }
+  }
+  // ── External routing ──────────────────────────────────────────────────────
+  async routeExternal(ctx, attachment, settings, defaultParser) {
+    var _a, _b, _c, _d, _e, _f;
+    if (!settings.activeProviderId) {
+      (_b = (_a = ctx.log) == null ? void 0 : _a.warn) == null ? void 0 : _b.call(_a, "[DocumentParser] mode=external but no activeProviderId configured");
+      return settings.fallbackToDefault ? defaultParser() : this.unsupportedResult(attachment);
+    }
+    const providerRecord = await this.getProvidersRepo().findById(settings.activeProviderId);
+    if (!providerRecord || !providerRecord.get("enabled")) {
+      (_d = (_c = ctx.log) == null ? void 0 : _c.warn) == null ? void 0 : _d.call(_c, `[DocumentParser] External provider ${settings.activeProviderId} not found or disabled`);
+      return settings.fallbackToDefault ? defaultParser() : this.unsupportedResult(attachment);
+    }
+    const providerConfig = this.recordToProviderConfig(providerRecord);
+    const supportedMimetypes = providerConfig["supportedMimetypes"] ?? [];
+    if (supportedMimetypes.length > 0 && attachment.mimetype && !supportedMimetypes.includes(attachment.mimetype)) {
+      return settings.fallbackToDefault ? defaultParser() : this.unsupportedResult(attachment);
+    }
+    try {
+      const { buffer, url } = await this.getFileBuffer(ctx, attachment);
+      const text = await (0, import_external_ocr_client.callExternalOcr)(providerConfig, {
+        fileBuffer: buffer,
+        filename: attachment.filename ?? attachment.name ?? "file",
+        mimetype: attachment.mimetype ?? "application/octet-stream",
+        fileUrl: url
+      });
+      return textToContentBlock(text, attachment);
+    } catch (err) {
+      (_f = (_e = ctx.log) == null ? void 0 : _e.warn) == null ? void 0 : _f.call(_e, `[DocumentParser] external OCR failed for "${attachment.filename}": ${err}`);
+      if (settings.fallbackToDefault) {
+        return defaultParser();
+      }
+      throw err;
+    }
+  }
+  // ── Helpers ───────────────────────────────────────────────────────────────
+  recordToProviderConfig(record) {
+    return {
+      apiEndpoint: record.get("apiEndpoint"),
+      authType: record.get("authType"),
+      apiKey: record.get("apiKey"),
+      authConfig: record.get("authConfig") ?? {},
+      requestFormat: record.get("requestFormat") ?? "multipart",
+      requestConfig: record.get("requestConfig") ?? {},
+      responseTextPath: record.get("responseTextPath") ?? "text",
+      timeout: record.get("timeout") ?? 6e4,
+      supportedMimetypes: record.get("supportedMimetypes") ?? []
+    };
+  }
+  unsupportedResult(attachment) {
+    return {
+      placement: "contentBlocks",
+      content: {
+        type: "text",
+        text: `[Attachment: ${attachment.filename ?? attachment.name ?? "file"} \u2014 no parser available]`
+      }
+    };
+  }
+}
+function resolveExtname(attachment) {
+  if (attachment.extname) return attachment.extname.toLowerCase();
+  const name = attachment.filename ?? attachment.name ?? "";
+  const idx = name.lastIndexOf(".");
+  return idx >= 0 ? name.slice(idx).toLowerCase() : "";
+}
+function docpixieReferenceBlock(documentId, filename, mimetype) {
+  const text = [
+    `<document_indexed filename="${filename}" type="${mimetype}" docpixie_id="${documentId}">`,
+    `This document has been submitted to DocPixie for deep indexing (Document ID: ${documentId}).`,
+    ``,
+    `IMPORTANT: Do NOT attempt to read the raw file content inline.`,
+    `Instead, use the \`docpixie:query\` tool to retrieve information from this document.`,
+    ``,
+    `Usage examples:`,
+    `  - Summarize: docpixie:query { "query": "summarize this document", "documentIds": [${documentId}] }`,
+    `  - Find info: docpixie:query { "query": "<user question>", "documentIds": [${documentId}] }`,
+    ``,
+    `Note: Indexing runs in the background. If you query immediately and get no results,`,
+    `wait a moment and retry \u2014 complex documents (PDF with many pages) take longer to index.`,
+    `</document_indexed>`
+  ].join("\n");
+  return {
+    placement: "contentBlocks",
+    content: { type: "text", text }
+  };
+}
+function textToContentBlock(text, attachment) {
+  const filename = attachment.filename ?? attachment.name ?? "document";
+  const mimetype = attachment.mimetype ?? "";
+  return {
+    placement: "contentBlocks",
+    content: {
+      type: "text",
+      text: `<document filename="${filename}" type="${mimetype}">
+${text}
+</document>`
+    }
+  };
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  ParseRouter
+});

package/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "plugin-document-parser",
+  "displayName": "Document Parser",
+  "displayName.zh-CN": "文档解析器",
+  "description": "Intercept and route AI chat file attachments through configurable internal or external OCR/parse providers.",
+  "version": "1.0.1",
+  "main": "dist/server/index.js",
+  "files": [
+    "dist",
+    "client.js",
+    "client.d.ts",
+    "server.js",
+    "server.d.ts",
+    "src"
+  ],
+  "nocobase": {
+    "supportedVersions": ["2.x"]
+  },
+  "peerDependencies": {
+    "@nocobase/client": "2.x",
+    "@nocobase/server": "2.x",
+    "@nocobase/database": "2.x",
+    "@nocobase/plugin-ai": "2.x",
+    "@nocobase/plugin-file-manager": "2.x",
+    "@nocobase/test": "2.x"
+  },
+  "dependencies": {
+    "axios": "^1.6.0",
+    "form-data": "^4.0.0"
+  },
+  "keywords": ["AI", "document", "parser", "OCR"],
+  "license": "Apache-2.0"
+}

package/server.d.ts

@@ -0,0 +1,2 @@
+export * from './dist/server';
+export { default } from './dist/server';

package/server.js

@@ -0,0 +1 @@
+module.exports = require('./dist/server/index.js');