plugin-knowledge-base 1.0.0

package/dist/server/pipeline/text-splitter.js

@@ -0,0 +1,69 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var text_splitter_exports = {};
+__export(text_splitter_exports, {
+  DocumentTextSplitter: () => DocumentTextSplitter
+});
+module.exports = __toCommonJS(text_splitter_exports);
+var import_textsplitters = require("@langchain/textsplitters");
+const DEFAULT_CHUNK_SIZE = 1e3;
+const DEFAULT_CHUNK_OVERLAP = 200;
+class DocumentTextSplitter {
+  splitter;
+  constructor(options) {
+    const {
+      chunkSize = DEFAULT_CHUNK_SIZE,
+      chunkOverlap = DEFAULT_CHUNK_OVERLAP,
+      separators
+    } = options ?? {};
+    this.splitter = new import_textsplitters.RecursiveCharacterTextSplitter({
+      chunkSize,
+      chunkOverlap,
+      ...separators ? { separators } : {}
+    });
+  }
+  async splitText(text, metadata) {
+    if (!text || text.trim().length === 0) {
+      return [];
+    }
+    const documents = await this.splitter.createDocuments(
+      [text],
+      [metadata ?? {}]
+    );
+    return documents;
+  }
+  async splitDocuments(documents) {
+    if (!documents || documents.length === 0) {
+      return [];
+    }
+    return this.splitter.splitDocuments(documents);
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  DocumentTextSplitter
+});

package/dist/server/pipeline/vectorization.js

@@ -0,0 +1,144 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var vectorization_exports = {};
+__export(vectorization_exports, {
+  VectorizationPipeline: () => VectorizationPipeline
+});
+module.exports = __toCommonJS(vectorization_exports);
+var import_text_splitter = require("./text-splitter");
+class VectorizationPipeline {
+  constructor(plugin) {
+    this.plugin = plugin;
+  }
+  async processDocument(documentId, options) {
+    const docRepo = this.plugin.db.getRepository("aiKnowledgeBaseDocuments");
+    await docRepo.update({
+      filter: { id: documentId },
+      values: { status: "processing" }
+    });
+    try {
+      const docRecord = await docRepo.findOne({
+        filter: { id: documentId },
+        appends: ["file", "knowledgeBase", "knowledgeBase.vectorStore", "knowledgeBase.vectorStore.vectorDatabase"]
+      });
+      if (!docRecord) {
+        throw new Error(`Document "${documentId}" not found`);
+      }
+      const doc = docRecord.toJSON();
+      const file = doc.file;
+      const knowledgeBase = doc.knowledgeBase;
+      if (!file) {
+        throw new Error("Document has no associated file");
+      }
+      if (!(knowledgeBase == null ? void 0 : knowledgeBase.vectorStore)) {
+        throw new Error("Knowledge base has no vector store configured");
+      }
+      const aiPlugin = this.plugin.aiPlugin;
+      const parseResult = await aiPlugin.documentLoaders.cached.load(file);
+      if (!parseResult.supported) {
+        throw new Error(`File type not supported: ${file.filename}`);
+      }
+      if (!parseResult.text || parseResult.text.trim().length === 0) {
+        await docRepo.update({
+          filter: { id: documentId },
+          values: { status: "success", chunkCount: 0 }
+        });
+        return { success: true, chunkCount: 0 };
+      }
+      const splitter = new import_text_splitter.DocumentTextSplitter(options);
+      const chunks = await splitter.splitText(parseResult.text, {
+        knowledgeBaseId: knowledgeBase.id,
+        knowledgeBaseOuterId: knowledgeBase.id,
+        documentId,
+        source: file.filename,
+        // Permission metadata for vector-level filtering
+        userId: doc.uploadedById ?? null,
+        accessLevel: knowledgeBase.accessLevel ?? "PUBLIC"
+      });
+      if (chunks.length === 0) {
+        await docRepo.update({
+          filter: { id: documentId },
+          values: { status: "success", chunkCount: 0 }
+        });
+        return { success: true, chunkCount: 0 };
+      }
+      const vectorStore = knowledgeBase.vectorStore;
+      const vectorDatabase = vectorStore.vectorDatabase;
+      const llmServiceRecord = await this.plugin.db.getRepository("llmServices").findOne({
+        filter: { name: vectorStore.llmService }
+      });
+      if (!llmServiceRecord) {
+        throw new Error(`LLM service "${vectorStore.llmService}" not found`);
+      }
+      const llmService = llmServiceRecord.toJSON();
+      const providerMeta = aiPlugin.aiManager.llmProviders.get(llmService.provider);
+      if (!(providerMeta == null ? void 0 : providerMeta.embedding)) {
+        throw new Error(
+          `LLM provider "${llmService.provider}" does not support embedding`
+        );
+      }
+      const embeddingProvider = new providerMeta.embedding({
+        app: this.plugin.app,
+        serviceOptions: llmService.options,
+        modelOptions: { model: vectorStore.embeddingModel }
+      });
+      const embeddings = embeddingProvider.createEmbedding();
+      const vdbProvider = aiPlugin.features.vectorDatabaseProvider;
+      const vectorStoreInstance = await vdbProvider.createVectorStore(
+        vectorDatabase.provider,
+        embeddings,
+        vectorDatabase.connectParams
+      );
+      await vectorStoreInstance.addDocuments(chunks);
+      await docRepo.update({
+        filter: { id: documentId },
+        values: {
+          status: "success",
+          chunkCount: chunks.length
+        }
+      });
+      return { success: true, chunkCount: chunks.length };
+    } catch (error) {
+      await docRepo.update({
+        filter: { id: documentId },
+        values: {
+          status: "failed",
+          error: error.message ?? String(error)
+        }
+      });
+      return {
+        success: false,
+        chunkCount: 0,
+        error: error.message ?? String(error)
+      };
+    }
+  }
+}
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  VectorizationPipeline
+});

package/dist/server/plugin.js

@@ -0,0 +1,131 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var plugin_exports = {};
+__export(plugin_exports, {
+  PluginKnowledgeBaseServer: () => PluginKnowledgeBaseServer,
+  default: () => plugin_default
+});
+module.exports = __toCommonJS(plugin_exports);
+var import_server = require("@nocobase/server");
+var import_plugin_ai = __toESM(require("@nocobase/plugin-ai"));
+var import_knowledge_base_impl = require("./features/knowledge-base-impl");
+var import_vector_database_impl = require("./features/vector-database-impl");
+var import_vector_database_provider_impl = require("./features/vector-database-provider-impl");
+var import_vector_store_provider_impl = require("./features/vector-store-provider-impl");
+var import_pgvector = require("./providers/pgvector");
+var import_vectorization = require("./pipeline/vectorization");
+var import_ai_knowledge_base = __toESM(require("./resources/ai-knowledge-base"));
+var import_ai_knowledge_base_documents = __toESM(require("./resources/ai-knowledge-base-documents"));
+var import_ai_vector_stores = __toESM(require("./resources/ai-vector-stores"));
+var import_ai_vector_databases = __toESM(require("./resources/ai-vector-databases"));
+class PluginKnowledgeBaseServer extends import_server.Plugin {
+  vectorizationPipeline;
+  _aiPlugin;
+  get aiPlugin() {
+    if (!this._aiPlugin) {
+      this._aiPlugin = this.pm.get(import_plugin_ai.default);
+    }
+    return this._aiPlugin;
+  }
+  async afterAdd() {
+  }
+  async beforeLoad() {
+  }
+  async load() {
+    const vdbProvider = new import_vector_database_provider_impl.VectorDatabaseProviderImpl();
+    vdbProvider.register(import_pgvector.pgVectorProviderInfo);
+    const vectorStoreProvider = new import_vector_store_provider_impl.VectorStoreProviderImpl(this, this.aiPlugin);
+    const vectorDatabase = new import_vector_database_impl.VectorDatabaseFeatureImpl(this);
+    const knowledgeBase = new import_knowledge_base_impl.KnowledgeBaseFeatureImpl(this);
+    this.aiPlugin.features.enableFeatures({
+      vectorDatabase,
+      vectorDatabaseProvider: vdbProvider,
+      vectorStoreProvider,
+      knowledgeBase
+    });
+    this.vectorizationPipeline = new import_vectorization.VectorizationPipeline(this);
+    this.defineResources();
+    this.setPermissions();
+    this.app.resourceManager.use(
+      async (ctx, next) => {
+        var _a;
+        const { resourceName, actionName } = ctx.action;
+        if (resourceName === "aiFiles" && actionName === "create") {
+          const storageName = (_a = ctx.action.params) == null ? void 0 : _a.storageRule;
+          if (storageName) {
+            const collection = ctx.db.getCollection("aiFiles");
+            collection.options.storage = storageName;
+          }
+        }
+        await next();
+      },
+      { before: "createMiddleware" }
+    );
+  }
+  defineResources() {
+    this.app.resourceManager.define(import_ai_knowledge_base.default);
+    this.app.resourceManager.define(import_ai_knowledge_base_documents.default);
+    this.app.resourceManager.define(import_ai_vector_stores.default);
+    this.app.resourceManager.define(import_ai_vector_databases.default);
+  }
+  setPermissions() {
+    this.app.acl.registerSnippet({
+      name: `pm.${this.name}.knowledge-base`,
+      actions: [
+        "aiKnowledgeBase:*",
+        "aiKnowledgeBaseDoc:*",
+        "aiVectorStore:*",
+        "aiVectorDatabase:*"
+      ]
+    });
+    this.app.acl.allow("aiKnowledgeBase", "list", "loggedIn");
+    this.app.acl.allow("aiKnowledgeBase", "get", "loggedIn");
+  }
+  async install() {
+  }
+  async afterEnable() {
+  }
+  async afterDisable() {
+  }
+  async remove() {
+  }
+}
+var plugin_default = PluginKnowledgeBaseServer;
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PluginKnowledgeBaseServer
+});

package/dist/server/providers/pgvector.js

@@ -0,0 +1,134 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __create = Object.create;
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __getProtoOf = Object.getPrototypeOf;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toESM = (mod, isNodeMode, target) => (target = mod != null ? __create(__getProtoOf(mod)) : {}, __copyProps(
+  // If the importer is in node compatibility mode or this is not an ESM
+  // file that has been converted to a CommonJS file using a Babel-
+  // compatible transform (i.e. "__esModule" has not been set), then set
+  // "default" to the CommonJS "module.exports" for node compatibility.
+  isNodeMode || !mod || !mod.__esModule ? __defProp(target, "default", { value: mod, enumerable: true }) : target,
+  mod
+));
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var pgvector_exports = {};
+__export(pgvector_exports, {
+  PGVectorDatabaseProvider: () => PGVectorDatabaseProvider,
+  pgVectorProviderInfo: () => pgVectorProviderInfo
+});
+module.exports = __toCommonJS(pgvector_exports);
+class PGVectorDatabaseProvider {
+  validateConnectParams(connectParams) {
+    const { host, port, username, password, database, tableName } = connectParams || {};
+    if (!host) {
+      throw new Error("Host is required");
+    }
+    if (!port) {
+      throw new Error("Port is required");
+    }
+    if (!username) {
+      throw new Error("Username is required");
+    }
+    if (!password) {
+      throw new Error("Password is required");
+    }
+    if (!database) {
+      throw new Error("Database is required");
+    }
+    if (!tableName) {
+      throw new Error("Table name is required");
+    }
+  }
+  async testConnection(connectParams) {
+    this.validateConnectParams(connectParams);
+    let client;
+    try {
+      const { Client } = await import("pg");
+      client = new Client({
+        host: connectParams.host,
+        port: connectParams.port,
+        user: connectParams.username,
+        password: connectParams.password,
+        database: connectParams.database
+      });
+      await client.connect();
+      const result = await client.query(
+        "SELECT 1 FROM pg_extension WHERE extname = 'vector'"
+      );
+      if (result.rows.length === 0) {
+        return {
+          success: false,
+          error: "pgvector extension is not installed. Run: CREATE EXTENSION vector;"
+        };
+      }
+      return { success: true };
+    } catch (error) {
+      return {
+        success: false,
+        error: error.message ?? "Failed to connect to PostgreSQL"
+      };
+    } finally {
+      if (client) {
+        try {
+          await client.end();
+        } catch {
+        }
+      }
+    }
+  }
+  async createVectorStore(embeddings, connectParams) {
+    this.validateConnectParams(connectParams);
+    const { PGVectorStore } = await import("@langchain/community/vectorstores/pgvector");
+    const config = {
+      postgresConnectionOptions: {
+        host: connectParams.host,
+        port: connectParams.port,
+        user: connectParams.username,
+        password: connectParams.password,
+        database: connectParams.database
+      },
+      tableName: connectParams.tableName,
+      columns: {
+        idColumnName: "id",
+        vectorColumnName: "embedding",
+        contentColumnName: "content",
+        metadataColumnName: "metadata"
+      }
+    };
+    const vectorStore = await PGVectorStore.initialize(embeddings, config);
+    return vectorStore;
+  }
+}
+const pgVectorProviderInfo = {
+  name: "pgvector",
+  spec: "PGVector",
+  provider: new PGVectorDatabaseProvider()
+};
+// Annotate the CommonJS export names for ESM import in node:
+0 && (module.exports = {
+  PGVectorDatabaseProvider,
+  pgVectorProviderInfo
+});

package/dist/server/resources/ai-knowledge-base-documents.js

@@ -0,0 +1,134 @@
+/**
+ * This file is part of the NocoBase (R) project.
+ * Copyright (c) 2020-2024 NocoBase Co., Ltd.
+ * Authors: NocoBase Team.
+ *
+ * This project is dual-licensed under AGPL-3.0 and NocoBase Commercial License.
+ * For more information, please refer to: https://www.nocobase.com/agreement.
+ */
+
+var __defProp = Object.defineProperty;
+var __getOwnPropDesc = Object.getOwnPropertyDescriptor;
+var __getOwnPropNames = Object.getOwnPropertyNames;
+var __hasOwnProp = Object.prototype.hasOwnProperty;
+var __export = (target, all) => {
+  for (var name in all)
+    __defProp(target, name, { get: all[name], enumerable: true });
+};
+var __copyProps = (to, from, except, desc) => {
+  if (from && typeof from === "object" || typeof from === "function") {
+    for (let key of __getOwnPropNames(from))
+      if (!__hasOwnProp.call(to, key) && key !== except)
+        __defProp(to, key, { get: () => from[key], enumerable: !(desc = __getOwnPropDesc(from, key)) || desc.enumerable });
+  }
+  return to;
+};
+var __toCommonJS = (mod) => __copyProps(__defProp({}, "__esModule", { value: true }), mod);
+var ai_knowledge_base_documents_exports = {};
+__export(ai_knowledge_base_documents_exports, {
+  default: () => ai_knowledge_base_documents_default
+});
+module.exports = __toCommonJS(ai_knowledge_base_documents_exports);
+var ai_knowledge_base_documents_default = {
+  // IMPORTANT: resource name must differ from collection name 'aiKnowledgeBaseDocuments'
+  // otherwise NocoBase's auto-registered collection CRUD overrides our custom handlers.
+  name: "aiKnowledgeBaseDoc",
+  actions: {
+    async list(ctx, next) {
+      const { filter, sort, page, pageSize } = ctx.action.params;
+      const repo = ctx.db.getRepository("aiKnowledgeBaseDocuments");
+      ctx.body = await repo.find({
+        filter,
+        sort: sort ?? ["-createdAt"],
+        limit: pageSize,
+        offset: page ? (page - 1) * (pageSize || 20) : 0,
+        appends: ["file"]
+      });
+      await next();
+    },
+    async create(ctx, next) {
+      var _a, _b, _c;
+      const rawValues = ctx.action.params.values || {};
+      const values = rawValues.values || rawValues;
+      const repo = ctx.db.getRepository("aiKnowledgeBaseDocuments");
+      const userId = (_b = (_a = ctx.auth) == null ? void 0 : _a.user) == null ? void 0 : _b.id;
+      const roles = ctx.state.currentRoles ?? [];
+      const isRoot = roles.includes("root");
+      if (values.knowledgeBaseId) {
+        const kbRepo = ctx.db.getRepository("aiKnowledgeBases");
+        const kb = await kbRepo.findOne({ filter: { id: values.knowledgeBaseId } });
+        if (kb) {
+          const kbData = kb.toJSON();
+          if (kbData.accessLevel === "BASIC" && kbData.ownerId !== userId) {
+            ctx.throw(403, "Only the owner can upload documents to a personal knowledge base");
+            return;
+          }
+          if (kbData.accessLevel === "PUBLIC" && !isRoot) {
+            ctx.throw(403, "Only administrators can upload documents to a public knowledge base");
+            return;
+          }
+          if (kbData.accessLevel === "SHARED") {
+            const canUpload = isRoot || ((_c = kbData.uploadRoles) == null ? void 0 : _c.some((r) => roles.includes(r)));
+            if (!canUpload) {
+              ctx.throw(403, "You do not have permission to upload documents to this knowledge base");
+              return;
+            }
+          }
+        }
+      }
+      values.uploadedById = userId;
+      const doc = await repo.create({ values });
+      const setClauses = [];
+      const bindValues = [];
+      let paramIdx = 1;
+      if (values.knowledgeBaseId) {
+        setClauses.push(`"knowledgeBaseId" = $${paramIdx++}`);
+        bindValues.push(values.knowledgeBaseId);
+      }
+      if (values.fileId) {
+        setClauses.push(`"fileId" = $${paramIdx++}`);
+        bindValues.push(values.fileId);
+      }
+      if (setClauses.length > 0) {
+        bindValues.push(doc.get("id"));
+        await ctx.db.sequelize.query(
+          `UPDATE "aiKnowledgeBaseDocuments" SET ${setClauses.join(", ")} WHERE id = $${paramIdx}`,
+          { bind: bindValues }
+        );
+        if (values.knowledgeBaseId) doc.set("knowledgeBaseId", values.knowledgeBaseId);
+        if (values.fileId) doc.set("fileId", values.fileId);
+      }
+      const plugin = ctx.app.pm.get("@nocobase/plugin-knowledge-base");
+      if (plugin == null ? void 0 : plugin.vectorizationPipeline) {
+        plugin.vectorizationPipeline.processDocument(doc.id).catch((err) => {
+          ctx.logger.error(`Vectorization failed for document ${doc.id}:`, err);
+        });
+      }
+      ctx.body = doc;
+      await next();
+    },
+    async destroy(ctx, next) {
+      const { filterByTk } = ctx.action.params;
+      const repo = ctx.db.getRepository("aiKnowledgeBaseDocuments");
+      await repo.destroy({ filterByTk });
+      ctx.body = { success: true };
+      await next();
+    },
+    async reprocess(ctx, next) {
+      const { filterByTk } = ctx.action.params;
+      const repo = ctx.db.getRepository("aiKnowledgeBaseDocuments");
+      await repo.update({
+        filterByTk,
+        values: { status: "pending", error: null, chunkCount: 0 }
+      });
+      const plugin = ctx.app.pm.get("@nocobase/plugin-knowledge-base");
+      if (plugin == null ? void 0 : plugin.vectorizationPipeline) {
+        plugin.vectorizationPipeline.processDocument(filterByTk).catch((err) => {
+          ctx.logger.error(`Re-vectorization failed for document ${filterByTk}:`, err);
+        });
+      }
+      ctx.body = { success: true };
+      await next();
+    }
+  }
+};