totem-llm 0.1.0

package/server/endpoints/api/document/index.js

@@ -0,0 +1,1210 @@
+const { validApiKey } = require("../../../utils/middleware/validApiKey");
+const { handleAPIFileUpload } = require("../../../utils/files/multer");
+const {
+  viewLocalFiles,
+  findDocumentInDocuments,
+  getDocumentsByFolder,
+  normalizePath,
+  isWithin,
+} = require("../../../utils/files");
+const { reqBody, safeJsonParse } = require("../../../utils/http");
+const { EventLogs } = require("../../../models/eventLogs");
+const { CollectorApi } = require("../../../utils/collectorApi");
+const fs = require("fs");
+const path = require("path");
+const { Document } = require("../../../models/documents");
+const { purgeFolder } = require("../../../utils/files/purgeDocument");
+const createFilesLib = require("../../../utils/agents/aibitat/plugins/create-files/lib");
+const documentsPath =
+  process.env.NODE_ENV === "development"
+    ? path.resolve(__dirname, "../../../storage/documents")
+    : path.resolve(process.env.STORAGE_DIR, `documents`);
+
+/**
+ * Runs a simple validation check on the addToWorkspaces query parameter to ensure it is a string of comma-separated workspace slugs.
+ * @param {*} request
+ * @param {*} response
+ * @param {*} next
+ * @returns
+ */
+function validateWorkspaceSlugQuery(request, response, next) {
+  const { addToWorkspaces = "" } = reqBody(request);
+  if (!addToWorkspaces) return next();
+  if (typeof addToWorkspaces !== "string") {
+    return response
+      .status(422)
+      .json({
+        success: false,
+        error: `addToWorkspaces must be a string of comma-separated workspace slugs. Got ${typeof addToWorkspaces}`,
+      })
+      .end();
+  }
+  next();
+}
+
+function apiDocumentEndpoints(app) {
+  if (!app) return;
+
+  app.post(
+    "/v1/document/upload",
+    [validApiKey, handleAPIFileUpload, validateWorkspaceSlugQuery],
+    async (request, response) => {
+      /*
+    #swagger.tags = ['Documents']
+    #swagger.description = 'Upload a new file to AnythingLLM to be parsed and prepared for embedding, with optional metadata.'
+    #swagger.requestBody = {
+      description: 'File to be uploaded.',
+      required: true,
+      content: {
+        "multipart/form-data": {
+          schema: {
+            type: 'object',
+            required: ['file'],
+            properties: {
+              file: {
+                type: 'string',
+                format: 'binary',
+                description: 'The file to upload'
+              },
+              addToWorkspaces: {
+                type: 'string',
+                description: 'comma-separated text-string of workspace slugs to embed the document into post-upload. eg: workspace1,workspace2',
+              },
+              metadata: {
+                type: 'object',
+                description: 'Key:Value pairs of metadata to attach to the document in JSON Object format. Only specific keys are allowed - see example.',
+                example: { 'title': 'Custom Title', 'docAuthor': 'Author Name', 'description': 'A brief description', 'docSource': 'Source of the document' }
+              }
+            },
+            required: ['file']
+          }
+        }
+      }
+    }
+    #swagger.responses[200] = {
+      content: {
+        "application/json": {
+          schema: {
+            type: 'object',
+            example: {
+              success: true,
+              error: null,
+              documents: [
+                {
+                  "location": "custom-documents/anythingllm.txt-6e8be64c-c162-4b43-9997-b068c0071e8b.json",
+                  "name": "anythingllm.txt-6e8be64c-c162-4b43-9997-b068c0071e8b.json",
+                  "url": "file:///Users/tim/Documents/anything-llm/collector/hotdir/anythingllm.txt",
+                  "title": "anythingllm.txt",
+                  "docAuthor": "Unknown",
+                  "description": "Unknown",
+                  "docSource": "a text file uploaded by the user.",
+                  "chunkSource": "anythingllm.txt",
+                  "published": "1/16/2024, 3:07:00 PM",
+                  "wordCount": 93,
+                  "token_count_estimate": 115,
+                }
+              ]
+            }
+          }
+        }
+      }
+    }
+    #swagger.responses[403] = {
+      schema: {
+        "$ref": "#/definitions/InvalidAPIKey"
+      }
+    }
+    */
+      try {
+        const Collector = new CollectorApi();
+        const { originalname } = request.file;
+        const { addToWorkspaces = "", metadata: _metadata = {} } =
+          reqBody(request);
+        const metadata =
+          typeof _metadata === "string"
+            ? safeJsonParse(_metadata, {})
+            : _metadata;
+        const processingOnline = await Collector.online();
+
+        if (!processingOnline) {
+          response
+            .status(500)
+            .json({
+              success: false,
+              error: `Document processing API is not online. Document ${originalname} will not be processed automatically.`,
+            })
+            .end();
+          return;
+        }
+
+        const { success, reason, documents } = await Collector.processDocument(
+          originalname,
+          metadata
+        );
+
+        if (!success) {
+          return response
+            .status(500)
+            .json({ success: false, error: reason, documents })
+            .end();
+        }
+
+        Collector.log(
+          `Document ${originalname} uploaded processed and successfully. It is now available in documents.`
+        );
+        await EventLogs.logEvent("api_document_uploaded", {
+          documentName: originalname,
+        });
+
+        if (!!addToWorkspaces)
+          await Document.api.uploadToWorkspace(
+            addToWorkspaces,
+            documents?.[0].location
+          );
+        response.status(200).json({ success: true, error: null, documents });
+      } catch (e) {
+        console.error(e.message, e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.post(
+    "/v1/document/upload/:folderName",
+    [validApiKey, handleAPIFileUpload, validateWorkspaceSlugQuery],
+    async (request, response) => {
+      /*
+      #swagger.tags = ['Documents']
+      #swagger.description = 'Upload a new file to a specific folder in AnythingLLM to be parsed and prepared for embedding. If the folder does not exist, it will be created.'
+      #swagger.parameters['folderName'] = {
+        in: 'path',
+        description: 'Target folder path (defaults to \"custom-documents\" if not provided)',
+        required: true,
+        type: 'string',
+        example: 'my-folder'
+      }
+      #swagger.requestBody = {
+        description: 'File to be uploaded, with optional metadata.',
+        required: true,
+        content: {
+          "multipart/form-data": {
+            schema: {
+              type: 'object',
+              required: ['file'],
+              properties: {
+                file: {
+                  type: 'string',
+                  format: 'binary',
+                  description: 'The file to upload'
+                },
+                addToWorkspaces: {
+                  type: 'string',
+                  description: 'comma-separated text-string of workspace slugs to embed the document into post-upload. eg: workspace1,workspace2',
+                },
+                metadata: {
+                  type: 'object',
+                  description: 'Key:Value pairs of metadata to attach to the document in JSON Object format. Only specific keys are allowed - see example.',
+                  example: { 'title': 'Custom Title', 'docAuthor': 'Author Name', 'description': 'A brief description', 'docSource': 'Source of the document' }
+                }
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[200] = {
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                success: true,
+                error: null,
+                documents: [{
+                  "location": "custom-documents/anythingllm.txt-6e8be64c-c162-4b43-9997-b068c0071e8b.json",
+                  "name": "anythingllm.txt-6e8be64c-c162-4b43-9997-b068c0071e8b.json",
+                  "url": "file:///Users/tim/Documents/anything-llm/collector/hotdir/anythingllm.txt",
+                  "title": "anythingllm.txt",
+                  "docAuthor": "Unknown",
+                  "description": "Unknown",
+                  "docSource": "a text file uploaded by the user.",
+                  "chunkSource": "anythingllm.txt",
+                  "published": "1/16/2024, 3:07:00 PM",
+                  "wordCount": 93,
+                  "token_count_estimate": 115
+                }]
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[403] = {
+        schema: {
+          "$ref": "#/definitions/InvalidAPIKey"
+        }
+      }
+      #swagger.responses[500] = {
+        description: "Internal Server Error",
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                success: false,
+                error: "Document processing API is not online. Document will not be processed automatically."
+              }
+            }
+          }
+        }
+      }
+      */
+      try {
+        const { originalname } = request.file;
+        const { addToWorkspaces = "", metadata: _metadata = {} } =
+          reqBody(request);
+        const metadata =
+          typeof _metadata === "string"
+            ? safeJsonParse(_metadata, {})
+            : _metadata;
+
+        let folder = request.params?.folderName || "custom-documents";
+        folder = normalizePath(folder);
+        const targetFolderPath = path.join(documentsPath, folder);
+
+        if (
+          !isWithin(path.resolve(documentsPath), path.resolve(targetFolderPath))
+        )
+          throw new Error("Invalid folder name");
+        if (!fs.existsSync(targetFolderPath))
+          fs.mkdirSync(targetFolderPath, { recursive: true });
+
+        const Collector = new CollectorApi();
+        const processingOnline = await Collector.online();
+        if (!processingOnline) {
+          return response
+            .status(500)
+            .json({
+              success: false,
+              error: `Document processing API is not online. Document ${originalname} will not be processed automatically.`,
+            })
+            .end();
+        }
+
+        // Process the uploaded document with metadata
+        const { success, reason, documents } = await Collector.processDocument(
+          originalname,
+          metadata
+        );
+        if (!success) {
+          return response
+            .status(500)
+            .json({ success: false, error: reason, documents })
+            .end();
+        }
+
+        // For each processed document, check if it is already in the desired folder.
+        // If not, move it using similar logic as in the move-files endpoint.
+        for (const doc of documents) {
+          const currentFolder = path.dirname(doc.location);
+          if (currentFolder !== folder) {
+            const sourcePath = path.join(
+              documentsPath,
+              normalizePath(doc.location)
+            );
+            const destinationPath = path.join(
+              targetFolderPath,
+              path.basename(doc.location)
+            );
+
+            if (
+              !isWithin(documentsPath, sourcePath) ||
+              !isWithin(documentsPath, destinationPath)
+            )
+              throw new Error("Invalid file location");
+
+            fs.renameSync(sourcePath, destinationPath);
+            doc.location = path.join(folder, path.basename(doc.location));
+            doc.name = path.basename(doc.location);
+          }
+        }
+
+        Collector.log(
+          `Document ${originalname} uploaded, processed, and moved to folder ${folder} successfully.`
+        );
+
+        await EventLogs.logEvent("api_document_uploaded", {
+          documentName: originalname,
+          folder,
+        });
+
+        if (!!addToWorkspaces)
+          await Document.api.uploadToWorkspace(
+            addToWorkspaces,
+            documents?.[0].location
+          );
+        response.status(200).json({ success: true, error: null, documents });
+      } catch (e) {
+        console.error(e.message, e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.post(
+    "/v1/document/upload-link",
+    [validApiKey, validateWorkspaceSlugQuery],
+    async (request, response) => {
+      /*
+    #swagger.tags = ['Documents']
+    #swagger.description = 'Upload a valid URL for AnythingLLM to scrape and prepare for embedding. Optionally, specify a comma-separated list of workspace slugs to embed the document into post-upload.'
+    #swagger.requestBody = {
+      description: 'Link of web address to be scraped and optionally a comma-separated list of workspace slugs to embed the document into post-upload, and optional metadata.',
+      required: true,
+      content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                "link": "https://anythingllm.com",
+                "addToWorkspaces": "workspace1,workspace2",
+                "scraperHeaders": {
+                  "Authorization": "Bearer token123",
+                  "My-Custom-Header": "value"
+                },
+                "metadata": {
+                  "title": "Custom Title",
+                  "docAuthor": "Author Name",
+                  "description": "A brief description",
+                  "docSource": "Source of the document"
+                }
+              }
+            }
+          }
+        }
+    }
+    #swagger.responses[200] = {
+      content: {
+        "application/json": {
+          schema: {
+            type: 'object',
+            example: {
+              success: true,
+              error: null,
+              documents: [
+                {
+                  "id": "c530dbe6-bff1-4b9e-b87f-710d539d20bc",
+                  "url": "file://useanything_com.html",
+                  "title": "useanything_com.html",
+                  "docAuthor": "no author found",
+                  "description": "No description found.",
+                  "docSource": "URL link uploaded by the user.",
+                  "chunkSource": "https:anythingllm.com.html",
+                  "published": "1/16/2024, 3:46:33 PM",
+                  "wordCount": 252,
+                  "pageContent": "AnythingLLM is the best....",
+                  "token_count_estimate": 447,
+                  "location": "custom-documents/url-useanything_com-c530dbe6-bff1-4b9e-b87f-710d539d20bc.json"
+                }
+              ]
+            }
+          }
+        }
+      }
+    }
+    #swagger.responses[403] = {
+      schema: {
+        "$ref": "#/definitions/InvalidAPIKey"
+      }
+    }
+    */
+      try {
+        const Collector = new CollectorApi();
+        const {
+          link,
+          addToWorkspaces = "",
+          scraperHeaders = {},
+          metadata: _metadata = {},
+        } = reqBody(request);
+        const metadata =
+          typeof _metadata === "string"
+            ? safeJsonParse(_metadata, {})
+            : _metadata;
+        const processingOnline = await Collector.online();
+
+        if (!processingOnline) {
+          return response
+            .status(500)
+            .json({
+              success: false,
+              error: `Document processing API is not online. Link ${link} will not be processed automatically.`,
+            })
+            .end();
+        }
+
+        const { success, reason, documents } = await Collector.processLink(
+          link,
+          scraperHeaders,
+          metadata
+        );
+        if (!success) {
+          return response
+            .status(500)
+            .json({ success: false, error: reason, documents })
+            .end();
+        }
+
+        Collector.log(
+          `Link ${link} uploaded processed and successfully. It is now available in documents.`
+        );
+        await EventLogs.logEvent("api_link_uploaded", {
+          link,
+        });
+
+        if (!!addToWorkspaces)
+          await Document.api.uploadToWorkspace(
+            addToWorkspaces,
+            documents?.[0].location
+          );
+        response.status(200).json({ success: true, error: null, documents });
+      } catch (e) {
+        console.error(e.message, e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.post(
+    "/v1/document/raw-text",
+    [validApiKey, validateWorkspaceSlugQuery],
+    async (request, response) => {
+      /*
+     #swagger.tags = ['Documents']
+     #swagger.description = 'Upload a file by specifying its raw text content and metadata values without having to upload a file.'
+     #swagger.requestBody = {
+      description: 'Text content and metadata of the file to be saved to the system. Use metadata-schema endpoint to get the possible metadata keys',
+      required: true,
+      content: {
+        "application/json": {
+          schema: {
+            type: 'object',
+            example: {
+              "textContent": "This is the raw text that will be saved as a document in AnythingLLM.",
+              "addToWorkspaces": "workspace1,workspace2",
+              "metadata": {
+                "title": "This key is required. See in /server/endpoints/api/document/index.js:287",
+                "keyOne": "valueOne",
+                "keyTwo": "valueTwo",
+                "etc": "etc"
+              }
+            }
+          }
+        }
+      }
+     }
+    #swagger.responses[200] = {
+      content: {
+        "application/json": {
+          schema: {
+            type: 'object',
+            example: {
+              success: true,
+              error: null,
+              documents: [
+                {
+                  "id": "c530dbe6-bff1-4b9e-b87f-710d539d20bc",
+                  "url": "file://my-document.txt",
+                  "title": "hello-world.txt",
+                  "docAuthor": "no author found",
+                  "description": "No description found.",
+                  "docSource": "My custom description set during upload",
+                  "chunkSource": "no chunk source specified",
+                  "published": "1/16/2024, 3:46:33 PM",
+                  "wordCount": 252,
+                  "pageContent": "AnythingLLM is the best....",
+                  "token_count_estimate": 447,
+                  "location": "custom-documents/raw-my-doc-text-c530dbe6-bff1-4b9e-b87f-710d539d20bc.json"
+                }
+              ]
+            }
+          }
+        }
+      }
+     }
+     #swagger.responses[403] = {
+       schema: {
+         "$ref": "#/definitions/InvalidAPIKey"
+       }
+     }
+     */
+      try {
+        const Collector = new CollectorApi();
+        const requiredMetadata = ["title"];
+        const {
+          textContent,
+          metadata: _metadata = {},
+          addToWorkspaces = "",
+        } = reqBody(request);
+        const metadata =
+          typeof _metadata === "string"
+            ? safeJsonParse(_metadata, {})
+            : _metadata;
+        const processingOnline = await Collector.online();
+
+        if (!processingOnline) {
+          return response
+            .status(500)
+            .json({
+              success: false,
+              error: `Document processing API is not online. Request will not be processed.`,
+            })
+            .end();
+        }
+
+        if (
+          !requiredMetadata.every(
+            (reqKey) =>
+              Object.keys(metadata).includes(reqKey) && !!metadata[reqKey]
+          )
+        ) {
+          return response
+            .status(422)
+            .json({
+              success: false,
+              error: `You are missing required metadata key:value pairs in your request. Required metadata key:values are ${requiredMetadata
+                .map((v) => `'${v}'`)
+                .join(", ")}`,
+            })
+            .end();
+        }
+
+        if (!textContent || textContent?.length === 0) {
+          return response
+            .status(422)
+            .json({
+              success: false,
+              error: `The 'textContent' key cannot have an empty value.`,
+            })
+            .end();
+        }
+
+        const { success, reason, documents } = await Collector.processRawText(
+          textContent,
+          metadata
+        );
+        if (!success) {
+          return response
+            .status(500)
+            .json({ success: false, error: reason, documents })
+            .end();
+        }
+
+        Collector.log(
+          `Document created successfully. It is now available in documents.`
+        );
+        await EventLogs.logEvent("api_raw_document_uploaded");
+
+        if (!!addToWorkspaces)
+          await Document.api.uploadToWorkspace(
+            addToWorkspaces,
+            documents?.[0].location
+          );
+        response.status(200).json({ success: true, error: null, documents });
+      } catch (e) {
+        console.error(e.message, e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.get("/v1/documents", [validApiKey], async (_, response) => {
+    /*
+    #swagger.tags = ['Documents']
+    #swagger.description = 'List of all locally-stored documents in instance'
+    #swagger.responses[200] = {
+      content: {
+        "application/json": {
+          schema: {
+            type: 'object',
+            example: {
+             "localFiles": {
+              "name": "documents",
+              "type": "folder",
+              items: [
+                {
+                  "name": "my-stored-document.json",
+                  "type": "file",
+                  "id": "bb07c334-4dab-4419-9462-9d00065a49a1",
+                  "url": "file://my-stored-document.txt",
+                  "title": "my-stored-document.txt",
+                  "cached": false
+                },
+              ]
+             }
+            }
+          }
+        }
+      }
+    }
+    #swagger.responses[403] = {
+      schema: {
+        "$ref": "#/definitions/InvalidAPIKey"
+      }
+    }
+    */
+    try {
+      const localFiles = await viewLocalFiles();
+      response.status(200).json({ localFiles });
+    } catch (e) {
+      console.error(e.message, e);
+      response.sendStatus(500).end();
+    }
+  });
+
+  app.get(
+    "/v1/documents/folder/:folderName",
+    [validApiKey],
+    async (request, response) => {
+      /*
+    #swagger.tags = ['Documents']
+    #swagger.description = 'Get all documents stored in a specific folder.'
+    #swagger.parameters['folderName'] = {
+      in: 'path',
+      description: 'Name of the folder to retrieve documents from',
+      required: true,
+      type: 'string'
+    }
+    #swagger.responses[200] = {
+      content: {
+        "application/json": {
+          schema: {
+            type: 'object',
+            example: {
+              folder: "custom-documents",
+              documents: [
+                {
+                  name: "document1.json",
+                  type: "file",
+                  cached: false,
+                  pinnedWorkspaces: [],
+                  watched: false,
+                  more: "data",
+                },
+                {
+                  name: "document2.json",
+                  type: "file",
+                  cached: false,
+                  pinnedWorkspaces: [],
+                  watched: false,
+                  more: "data",
+                },
+              ]
+            }
+          }
+        }
+      }
+    }
+    #swagger.responses[403] = {
+      schema: {
+        "$ref": "#/definitions/InvalidAPIKey"
+      }
+    }
+    */
+      try {
+        const { folderName } = request.params;
+        const result = await getDocumentsByFolder(folderName);
+        response.status(result.code).json({
+          folder: result.folder,
+          documents: result.documents,
+          error: result.error,
+        });
+      } catch (e) {
+        console.error(e.message, e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.get(
+    "/v1/document/accepted-file-types",
+    [validApiKey],
+    async (_, response) => {
+      /*
+    #swagger.tags = ['Documents']
+    #swagger.description = 'Check available filetypes and MIMEs that can be uploaded.'
+    #swagger.responses[200] = {
+      content: {
+        "application/json": {
+          schema: {
+            type: 'object',
+            example: {
+              "types": {
+                "application/mbox": [
+                  ".mbox"
+                ],
+                "application/pdf": [
+                  ".pdf"
+                ],
+                "application/vnd.oasis.opendocument.text": [
+                  ".odt"
+                ],
+                "application/vnd.openxmlformats-officedocument.wordprocessingml.document": [
+                  ".docx"
+                ],
+                "text/plain": [
+                  ".txt",
+                  ".md"
+                ]
+              }
+            }
+          }
+        }
+      }
+    }
+    #swagger.responses[403] = {
+      schema: {
+        "$ref": "#/definitions/InvalidAPIKey"
+      }
+    }
+    */
+      try {
+        const types = await new CollectorApi().acceptedFileTypes();
+        if (!types) {
+          response.sendStatus(404).end();
+          return;
+        }
+
+        response.status(200).json({ types });
+      } catch (e) {
+        console.error(e.message, e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
+  app.get(
+    "/v1/document/metadata-schema",
+    [validApiKey],
+    async (_, response) => {
+      /*
+    #swagger.tags = ['Documents']
+    #swagger.description = 'Get the known available metadata schema for when doing a raw-text upload and the acceptable type of value for each key.'
+    #swagger.responses[200] = {
+      content: {
+        "application/json": {
+          schema: {
+            type: 'object',
+            example: {
+             "schema": {
+                "keyOne": "string | number | nullable",
+                "keyTwo": "string | number | nullable",
+                "specialKey": "number",
+                "title": "string",
+              }
+            }
+          }
+        }
+      }
+    }
+    #swagger.responses[403] = {
+      schema: {
+        "$ref": "#/definitions/InvalidAPIKey"
+      }
+    }
+    */
+      try {
+        response.status(200).json({
+          schema: {
+            // If you are updating this be sure to update the collector METADATA_KEYS constant in /processRawText.
+            url: "string | nullable",
+            title: "string",
+            docAuthor: "string | nullable",
+            description: "string | nullable",
+            docSource: "string | nullable",
+            chunkSource: "string | nullable",
+            published: "epoch timestamp in ms | nullable",
+          },
+        });
+      } catch (e) {
+        console.error(e.message, e);
+        response.sendStatus(500).end();
+      }
+    }
+  );
+
+  // Be careful and place as last route to prevent override of the other /document/ GET
+  // endpoints!
+  app.get("/v1/document/:docName", [validApiKey], async (request, response) => {
+    /*
+    #swagger.tags = ['Documents']
+    #swagger.description = 'Get a single document by its unique AnythingLLM document name'
+    #swagger.parameters['docName'] = {
+        in: 'path',
+        description: 'Unique document name to find (name in /documents)',
+        required: true,
+        type: 'string'
+    }
+    #swagger.responses[200] = {
+      content: {
+        "application/json": {
+          schema: {
+            type: 'object',
+            example: {
+             "localFiles": {
+              "name": "documents",
+              "type": "folder",
+              items: [
+                {
+                  "name": "my-stored-document.txt-uuid1234.json",
+                  "type": "file",
+                  "id": "bb07c334-4dab-4419-9462-9d00065a49a1",
+                  "url": "file://my-stored-document.txt",
+                  "title": "my-stored-document.txt",
+                  "cached": false
+                },
+              ]
+             }
+            }
+          }
+        }
+      }
+    }
+    #swagger.responses[403] = {
+      schema: {
+        "$ref": "#/definitions/InvalidAPIKey"
+      }
+    }
+    */
+    try {
+      const { docName } = request.params;
+      const document = await findDocumentInDocuments(docName);
+      if (!document) {
+        response.sendStatus(404).end();
+        return;
+      }
+      response.status(200).json({ document });
+    } catch (e) {
+      console.error(e.message, e);
+      response.sendStatus(500).end();
+    }
+  });
+
+  app.post(
+    "/v1/document/create-folder",
+    [validApiKey],
+    async (request, response) => {
+      /*
+      #swagger.tags = ['Documents']
+      #swagger.description = 'Create a new folder inside the documents storage directory.'
+      #swagger.requestBody = {
+        description: 'Name of the folder to create.',
+        required: true,
+        content: {
+          "application/json": {
+            schema: {
+              type: 'string',
+              example: {
+                "name": "new-folder"
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[200] = {
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                success: true,
+                message: null
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[403] = {
+        schema: {
+          "$ref": "#/definitions/InvalidAPIKey"
+        }
+      }
+      */
+      try {
+        const { name } = reqBody(request);
+        const storagePath = path.join(documentsPath, normalizePath(name));
+        if (!isWithin(path.resolve(documentsPath), path.resolve(storagePath)))
+          throw new Error("Invalid path name");
+
+        if (fs.existsSync(storagePath)) {
+          response.status(500).json({
+            success: false,
+            message: "Folder by that name already exists",
+          });
+          return;
+        }
+
+        fs.mkdirSync(storagePath, { recursive: true });
+        response.status(200).json({ success: true, message: null });
+      } catch (e) {
+        console.error(e);
+        response.status(500).json({
+          success: false,
+          message: `Failed to create folder: ${e.message}`,
+        });
+      }
+    }
+  );
+
+  app.delete(
+    "/v1/document/remove-folder",
+    [validApiKey],
+    async (request, response) => {
+      /*
+      #swagger.tags = ['Documents']
+      #swagger.description = 'Remove a folder and all its contents from the documents storage directory.'
+      #swagger.requestBody = {
+        description: 'Name of the folder to remove.',
+        required: true,
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              properties: {
+                name: {
+                  type: 'string',
+                  example: "my-folder"
+                }
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[200] = {
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                success: true,
+                message: "Folder removed successfully"
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[403] = {
+        schema: {
+          "$ref": "#/definitions/InvalidAPIKey"
+        }
+      }
+      */
+      try {
+        const { name } = reqBody(request);
+        await purgeFolder(name);
+        response
+          .status(200)
+          .json({ success: true, message: "Folder removed successfully" });
+      } catch (e) {
+        console.error(e);
+        response.status(500).json({
+          success: false,
+          message: `Failed to remove folder: ${e.message}`,
+        });
+      }
+    }
+  );
+
+  app.post(
+    "/v1/document/move-files",
+    [validApiKey],
+    async (request, response) => {
+      /*
+      #swagger.tags = ['Documents']
+      #swagger.description = 'Move files within the documents storage directory.'
+      #swagger.requestBody = {
+        description: 'Array of objects containing source and destination paths of files to move.',
+        required: true,
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                "files": [
+                  {
+                    "from": "custom-documents/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json",
+                    "to": "folder/file.txt-fc4beeeb-e436-454d-8bb4-e5b8979cb48f.json"
+                  }
+                ]
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[200] = {
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                success: true,
+                message: null
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[403] = {
+        schema: {
+          "$ref": "#/definitions/InvalidAPIKey"
+        }
+      }
+      */
+      try {
+        const { files } = reqBody(request);
+        const docpaths = files.map(({ from }) => from);
+        const documents = await Document.where({ docpath: { in: docpaths } });
+        const embeddedFiles = documents.map((doc) => doc.docpath);
+        const moveableFiles = files.filter(
+          ({ from }) => !embeddedFiles.includes(from)
+        );
+        const movePromises = moveableFiles.map(({ from, to }) => {
+          const sourcePath = path.join(documentsPath, normalizePath(from));
+          const destinationPath = path.join(documentsPath, normalizePath(to));
+          return new Promise((resolve, reject) => {
+            if (
+              !isWithin(documentsPath, sourcePath) ||
+              !isWithin(documentsPath, destinationPath)
+            )
+              return reject("Invalid file location");
+
+            fs.rename(sourcePath, destinationPath, (err) => {
+              if (err) {
+                console.error(`Error moving file ${from} to ${to}:`, err);
+                reject(err);
+              } else {
+                resolve();
+              }
+            });
+          });
+        });
+        Promise.all(movePromises)
+          .then(() => {
+            const unmovableCount = files.length - moveableFiles.length;
+            if (unmovableCount > 0) {
+              response.status(200).json({
+                success: true,
+                message: `${unmovableCount}/${files.length} files not moved. Unembed them from all workspaces.`,
+              });
+            } else {
+              response.status(200).json({
+                success: true,
+                message: null,
+              });
+            }
+          })
+          .catch((err) => {
+            console.error("Error moving files:", err);
+            response
+              .status(500)
+              .json({ success: false, message: "Failed to move some files." });
+          });
+      } catch (e) {
+        console.error(e);
+        response
+          .status(500)
+          .json({ success: false, message: "Failed to move files." });
+      }
+    }
+  );
+
+  app.get(
+    "/v1/document/generated-files/:filename",
+    [validApiKey],
+    async (request, response) => {
+      /*
+      #swagger.tags = ['Documents']
+      #swagger.description = 'Download a file generated by an agent skill (e.g., PDF, DOCX, XLSX, PPTX). The filename is returned in the `outputs` array of a chat response when an agent generates a file. Use the `storageFilename` value from that response to download the file.'
+      #swagger.parameters['filename'] = {
+        in: 'path',
+        description: 'The storage filename returned in the chat response outputs array (e.g., pdf-e9e14f28-d6b6-4f49-91a0-dd331517f567.pdf)',
+        required: true,
+        type: 'string'
+      }
+      #swagger.responses[200] = {
+        description: 'File downloaded successfully',
+        content: {
+          "application/octet-stream": {
+            schema: {
+              type: 'string',
+              format: 'binary'
+            }
+          }
+        }
+      }
+      #swagger.responses[400] = {
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                error: "Invalid filename format"
+              }
+            }
+          }
+        }
+      }
+      #swagger.responses[403] = {
+        schema: {
+          "$ref": "#/definitions/InvalidAPIKey"
+        }
+      }
+      #swagger.responses[404] = {
+        content: {
+          "application/json": {
+            schema: {
+              type: 'object',
+              example: {
+                error: "File not found"
+              }
+            }
+          }
+        }
+      }
+      */
+      try {
+        const { filename } = request.params;
+        if (!filename)
+          return response.status(400).json({ error: "Filename is required" });
+
+        const parsed = createFilesLib.parseFilename(filename);
+        if (!parsed) {
+          return response
+            .status(400)
+            .json({ error: "Invalid filename format" });
+        }
+
+        const fileData = await createFilesLib.getGeneratedFile(filename);
+        if (!fileData) {
+          return response.status(404).json({ error: "File not found" });
+        }
+
+        const mimeType = createFilesLib.getMimeType(`.${parsed.extension}`);
+        const safeFilename = createFilesLib.sanitizeFilenameForHeader(filename);
+        response.setHeader("Content-Type", mimeType);
+        response.setHeader(
+          "Content-Disposition",
+          `attachment; filename="${safeFilename}"`
+        );
+        response.setHeader("Content-Length", fileData.buffer.length);
+        response.send(fileData.buffer);
+
+      } catch (error) {
+        console.error(
+          "[document/generated-files] Download error:",
+          error.message
+        );
+        return response.status(500).json({ error: "Failed to download file" });
+      }
+    }
+  );
+}
+
+module.exports = { apiDocumentEndpoints };