totem-llm 0.1.0

package/collector/utils/extensions/DrupalWiki/index.js

@@ -0,0 +1,102 @@
+/**
+ * Copyright 2024
+ *
+ * Authors:
+ *  - Eugen Mayer (KontextWork)
+ */
+
+const { DrupalWiki } = require("./DrupalWiki");
+const { validBaseUrl } = require("../../../utils/http");
+
+async function loadAndStoreSpaces(
+  { baseUrl = null, spaceIds = null, accessToken = null },
+  response
+) {
+  if (!baseUrl) {
+    return {
+      success: false,
+      reason:
+        "Please provide your baseUrl like https://mywiki.drupal-wiki.net.",
+    };
+  } else if (!validBaseUrl(baseUrl)) {
+    return {
+      success: false,
+      reason: "Provided base URL is not a valid URL.",
+    };
+  }
+
+  if (!spaceIds) {
+    return {
+      success: false,
+      reason:
+        "Please provide a list of spaceIds like 21,56,67 you want to extract",
+    };
+  }
+
+  if (!accessToken) {
+    return {
+      success: false,
+      reason: "Please provide a REST API-Token.",
+    };
+  }
+
+  console.log(`-- Working Drupal Wiki ${baseUrl} for spaceIds: ${spaceIds} --`);
+  const drupalWiki = new DrupalWiki({ baseUrl, accessToken });
+
+  const encryptionWorker = response.locals.encryptionWorker;
+  const spaceIdsArr = spaceIds.split(",").map((idStr) => {
+    return Number(idStr.trim());
+  });
+
+  for (const spaceId of spaceIdsArr) {
+    try {
+      await drupalWiki.loadAndStoreAllPagesForSpace(spaceId, encryptionWorker);
+      console.log(`--- Finished space ${spaceId} ---`);
+    } catch (e) {
+      console.error(e);
+      return {
+        success: false,
+        reason: e.message,
+        data: {},
+      };
+    }
+  }
+  console.log(`-- Finished all spaces--`);
+
+  return {
+    success: true,
+    reason: null,
+    data: {
+      spaceIds,
+      destination: drupalWiki.storagePath,
+    },
+  };
+}
+
+/**
+ * Gets the page content from a specific Confluence page, not all pages in a workspace.
+ * @returns
+ */
+async function loadPage({ baseUrl, pageId, accessToken }) {
+  console.log(`-- Working Drupal Wiki Page ${pageId} of ${baseUrl} --`);
+  const drupalWiki = new DrupalWiki({ baseUrl, accessToken });
+  try {
+    const page = await drupalWiki.loadPage(pageId);
+    return {
+      success: true,
+      reason: null,
+      content: page.processedBody,
+    };
+  } catch {
+    return {
+      success: false,
+      reason: `Failed (re)-fetching DrupalWiki page ${pageId} form ${baseUrl}}`,
+      content: null,
+    };
+  }
+}
+
+module.exports = {
+  loadAndStoreSpaces,
+  loadPage,
+};

package/collector/utils/extensions/ObsidianVault/index.js

@@ -0,0 +1,95 @@
+const { v4 } = require("uuid");
+const { default: slugify } = require("slugify");
+const path = require("path");
+const fs = require("fs");
+const {
+  writeToServerDocuments,
+  sanitizeFileName,
+  documentsFolder,
+} = require("../../files");
+
+function parseObsidianVaultPath(files = []) {
+  const possiblePaths = new Set();
+  files.forEach(
+    (file) => file?.path && possiblePaths.add(file.path.split("/")[0])
+  );
+
+  switch (possiblePaths.size) {
+    case 0:
+      return null;
+    case 1:
+      // The user specified a vault properly - so all files are in the same folder.
+      return possiblePaths.values().next().value;
+    default:
+      return null;
+  }
+}
+
+async function loadObsidianVault({ files = [] }) {
+  if (!files || files?.length === 0)
+    return { success: false, error: "No files provided" };
+  const vaultName = parseObsidianVaultPath(files);
+  const folderUUId = v4().slice(0, 4);
+  const outFolder = vaultName
+    ? slugify(`obsidian-vault-${vaultName}-${folderUUId}`).toLowerCase()
+    : slugify(`obsidian-${folderUUId}`).toLowerCase();
+  const outFolderPath = path.resolve(documentsFolder, outFolder);
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
+
+  console.log(
+    `Processing ${files.length} files from Obsidian Vault ${
+      vaultName ? `"${vaultName}"` : ""
+    }`
+  );
+  const results = [];
+  for (const file of files) {
+    try {
+      const fullPageContent = file?.content;
+      // If the file has no content or is just whitespace, skip it.
+      if (!fullPageContent || fullPageContent.trim() === "") continue;
+
+      const data = {
+        id: v4(),
+        url: `obsidian://${file.path}`,
+        title: file.name,
+        docAuthor: "Obsidian Vault",
+        description: file.name,
+        docSource: "Obsidian Vault",
+        chunkSource: `obsidian://${file.path}`,
+        published: new Date().toLocaleString(),
+        wordCount: fullPageContent.split(" ").length,
+        pageContent: fullPageContent,
+        token_count_estimate: fullPageContent.length / 4, // rough estimate
+      };
+
+      const targetFileName = sanitizeFileName(
+        `${slugify(file.name)}-${data.id}`
+      );
+      writeToServerDocuments({
+        data,
+        filename: targetFileName,
+        destinationOverride: outFolderPath,
+      });
+      results.push({ file: file.path, status: "success" });
+    } catch (e) {
+      console.error(`Failed to process ${file.path}:`, e);
+      results.push({ file: file.path, status: "failed", reason: e.message });
+    }
+  }
+
+  return {
+    success: true,
+    data: {
+      processed: results.filter((r) => r.status === "success").length,
+      failed: results.filter((r) => r.status === "failed").length,
+      total: files.length,
+      results,
+      destination: path.basename(outFolderPath),
+    },
+  };
+}
+
+module.exports = {
+  loadObsidianVault,
+};

package/collector/utils/extensions/PaperlessNgx/PaperlessNgxLoader/index.js

@@ -0,0 +1,157 @@
+const { htmlToText } = require("html-to-text");
+const pdf = require("pdf-parse");
+
+class PaperlessNgxLoader {
+  constructor({ baseUrl, apiToken }) {
+    this.baseUrl = new URL(baseUrl).origin;
+    this.apiToken = apiToken;
+    this.baseHeaders = {
+      Authorization: `Token ${this.apiToken}`,
+    };
+  }
+
+  async load() {
+    try {
+      const documents = await this.fetchAllDocuments();
+      return documents.map((doc) => this.createDocumentFromPage(doc));
+    } catch (error) {
+      console.error("Error:", error);
+      throw error;
+    }
+  }
+
+  /**
+   * Fetches all documents from Paperless-ngx
+   * @returns {Promise<{{[key: string]: any, content: string}[]}>} The documents with their content
+   */
+  async fetchAllDocuments() {
+    try {
+      const documents = [];
+      let nextUrl = `${this.baseUrl}/api/documents/`;
+      let page = 1;
+
+      while (nextUrl) {
+        console.log(`Fetching documents page ${page} from Paperless-ngx`);
+        try {
+          const data = await fetch(nextUrl, {
+            headers: {
+              "Content-Type": "application/json",
+              ...this.baseHeaders,
+            },
+          }).then((res) => {
+            if (!res.ok)
+              throw new Error(
+                `Failed to fetch documents from Paperless-ngx: ${res.status}`
+              );
+            return res.json();
+          });
+
+          const validResults = data.results.filter((doc) => doc?.id);
+          if (!validResults.length) break;
+
+          documents.push(...validResults);
+
+          if (data.next === nextUrl) break;
+          nextUrl = data.next || null;
+          page++;
+        } catch (error) {
+          console.error(
+            `Error fetching page ${page} from Paperless-ngx:`,
+            error
+          );
+          break;
+        }
+      }
+
+      console.log(
+        `Fetched ${documents.length} documents from Paperless-ngx (Pages: ${
+          page - 1
+        })`
+      );
+
+      const documentsWithContent = await Promise.all(
+        documents.map(async (doc) => {
+          const content = await this.fetchDocumentContent(doc.id);
+          return { ...doc, content };
+        })
+      );
+
+      return documentsWithContent.filter((doc) => !!doc.content);
+    } catch (error) {
+      throw new Error(
+        `Failed to fetch documents from Paperless-ngx: ${error.message}`
+      );
+    }
+  }
+
+  /**
+   * Fetches the content of a document from Paperless-ngx
+   * @param {string} documentId - The ID of the document to fetch
+   * @returns {Promise<string>} The content of the document
+   */
+  async fetchDocumentContent(documentId) {
+    try {
+      const response = await fetch(
+        `${this.baseUrl}/api/documents/${documentId}/download/`,
+        {
+          headers: this.baseHeaders,
+        }
+      );
+
+      if (!response.ok)
+        throw new Error(`Failed to fetch document content: ${response.status}`);
+
+      const contentType = response.headers.get("content-type");
+      switch (contentType) {
+        case "text/plain":
+          return await response.text();
+        case "application/pdf":
+          const buffer = await response.arrayBuffer();
+          return await this.parsePdfContent(buffer);
+        default:
+          return await response.text();
+      }
+    } catch (error) {
+      console.error(
+        `Failed to fetch content for document ${documentId}:`,
+        error
+      );
+      return "";
+    }
+  }
+
+  async parsePdfContent(buffer) {
+    try {
+      const data = await pdf(Buffer.from(buffer));
+      return data.text;
+    } catch (error) {
+      console.error("Failed to parse PDF content:", error);
+      return "";
+    }
+  }
+
+  createDocumentFromPage(doc) {
+    const content = doc.content || "";
+    const plainTextContent = htmlToText(content, {
+      wordwrap: false,
+      preserveNewlines: true,
+    });
+
+    return {
+      pageContent: plainTextContent,
+      metadata: {
+        id: doc.id,
+        title: doc.original_file_name,
+        created: doc.created,
+        modified: doc.modified,
+        added: doc.added,
+        tags: doc.tags,
+        correspondent: doc.correspondent,
+        documentType: doc.document_type,
+        url: `${this.baseUrl}/documents/${doc.id}`,
+      },
+    };
+  }
+}
+
+module.exports = PaperlessNgxLoader;

package/collector/utils/extensions/PaperlessNgx/index.js

@@ -0,0 +1,128 @@
+const fs = require("fs");
+const path = require("path");
+const { default: slugify } = require("slugify");
+const { v4 } = require("uuid");
+const {
+  writeToServerDocuments,
+  sanitizeFileName,
+  documentsFolder,
+} = require("../../files");
+const { tokenizeString } = require("../../tokenizer");
+const { validBaseUrl } = require("../../http");
+const PaperlessNgxLoader = require("./PaperlessNgxLoader");
+
+/**
+ * Load documents from a Paperless-ngx instance
+ * @param {object} args - forwarded request body params
+ * @param {import("../../../middleware/setDataSigner").ResponseWithSigner} response - Express response object with encryptionWorker
+ * @returns
+ */
+async function loadPaperlessNgx({ baseUrl = null, apiToken = null }, response) {
+  if (!baseUrl || !validBaseUrl(baseUrl)) {
+    return {
+      success: false,
+      reason: "Provided base URL is not a valid URL.",
+    };
+  }
+
+  if (!apiToken) {
+    return {
+      success: false,
+      reason:
+        "You need to provide an API token to use the Paperless-ngx connector.",
+    };
+  }
+
+  const { origin, hostname } = new URL(baseUrl);
+  console.log(`-- Working Paperless-ngx ${origin} --`);
+  const loader = new PaperlessNgxLoader({
+    baseUrl: origin,
+    apiToken,
+  });
+
+  const { docs, error } = await loader
+    .load()
+    .then((docs) => ({ docs, error: null }))
+    .catch((e) => ({
+      docs: [],
+      error: e.message?.split("Error:")?.[1] || e.message,
+    }));
+
+  if (!docs.length || !!error) {
+    return {
+      success: false,
+      reason:
+        error ?? "No parseable documents found in that Paperless-ngx instance.",
+      data: null,
+    };
+  }
+
+  const outFolder = slugify(
+    `paperless-${hostname}-${v4().slice(0, 4)}`
+  ).toLowerCase();
+  const outFolderPath = path.resolve(documentsFolder, outFolder);
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
+
+  docs.forEach((doc) => {
+    if (!doc.pageContent) return;
+
+    const data = {
+      id: v4(),
+      url: doc.metadata.url,
+      title: doc.metadata.title,
+      docAuthor: doc.metadata.correspondent || "Unknown",
+      description: `A document from the Paperless-ngx instance at ${origin}`,
+      docSource: `paperless-ngx`,
+      chunkSource: generateChunkSource(
+        { doc, baseUrl: origin, apiToken },
+        response.locals.encryptionWorker
+      ),
+      published: doc.metadata.created,
+      wordCount: doc.pageContent.split(" ").length,
+      pageContent: doc.pageContent,
+      token_count_estimate: tokenizeString(doc.pageContent),
+    };
+
+    console.log(
+      `[Paperless-ngx Loader]: Saving ${doc.metadata.title} to ${outFolder}`
+    );
+    const fileName = sanitizeFileName(
+      `${slugify(doc.metadata.title)}-${data.id}`
+    );
+    writeToServerDocuments({
+      data,
+      filename: fileName,
+      destinationOverride: outFolderPath,
+    });
+  });
+
+  return {
+    success: true,
+    reason: null,
+    data: {
+      files: docs.length,
+      destination: outFolder,
+    },
+  };
+}
+
+/**
+ * Generate the full chunkSource for a specific Paperless-ngx document so that we can resync it later.
+ * @param {object} chunkSourceInformation
+ * @param {import("../../EncryptionWorker").EncryptionWorker} encryptionWorker
+ * @returns {string}
+ */
+function generateChunkSource({ doc, baseUrl, apiToken }, encryptionWorker) {
+  const payload = {
+    baseUrl,
+    token: apiToken,
+  };
+  return `paperless-ngx://${doc.metadata.id}?payload=${encryptionWorker.encrypt(
+    JSON.stringify(payload)
+  )}`;
+}
+
+module.exports = {
+  loadPaperlessNgx,
+};