totem-llm 0.1.0

package/collector/processSingleFile/convert/asOfficeMime.js

@@ -0,0 +1,59 @@
+const { v4 } = require("uuid");
+const officeParser = require("officeparser");
+const {
+  createdDate,
+  trashFile,
+  writeToServerDocuments,
+} = require("../../utils/files");
+const { tokenizeString } = require("../../utils/tokenizer");
+const { default: slugify } = require("slugify");
+
+async function asOfficeMime({
+  fullFilePath = "",
+  filename = "",
+  options = {},
+  metadata = {},
+}) {
+  console.log(`-- Working ${filename} --`);
+  let content = "";
+  try {
+    content = await officeParser.parseOfficeAsync(fullFilePath);
+  } catch (error) {
+    console.error(`Could not parse office or office-like file`, error);
+  }
+
+  if (!content.length) {
+    console.error(`Resulting text content was empty for ${filename}.`);
+    if (!options.absolutePath) trashFile(fullFilePath);
+    return {
+      success: false,
+      reason: `No text content found in ${filename}.`,
+      documents: [],
+    };
+  }
+
+  const data = {
+    id: v4(),
+    url: "file://" + fullFilePath,
+    title: metadata.title || filename,
+    docAuthor: metadata.docAuthor || "no author found",
+    description: metadata.description || "No description found.",
+    docSource: metadata.docSource || "Office file uploaded by the user.",
+    chunkSource: metadata.chunkSource || "",
+    published: createdDate(fullFilePath),
+    wordCount: content.split(" ").length,
+    pageContent: content,
+    token_count_estimate: tokenizeString(content),
+  };
+
+  const document = writeToServerDocuments({
+    data,
+    filename: `${slugify(filename)}-${data.id}`,
+    options: { parseOnly: options.parseOnly },
+  });
+  if (!options.absolutePath) trashFile(fullFilePath);
+  console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
+  return { success: true, reason: null, documents: [document] };
+}
+
+module.exports = asOfficeMime;

package/collector/processSingleFile/convert/asPDF/PDFLoader/index.js

@@ -0,0 +1,97 @@
+const fs = require("fs").promises;
+
+class PDFLoader {
+  constructor(filePath, { splitPages = true } = {}) {
+    this.filePath = filePath;
+    this.splitPages = splitPages;
+  }
+
+  async load() {
+    const buffer = await fs.readFile(this.filePath);
+    const { getDocument, version } = await this.getPdfJS();
+
+    const pdf = await getDocument({
+      data: new Uint8Array(buffer),
+      useWorkerFetch: false,
+      isEvalSupported: false,
+      useSystemFonts: true,
+    }).promise;
+
+    const meta = await pdf.getMetadata().catch(() => null);
+    const documents = [];
+
+    for (let i = 1; i <= pdf.numPages; i += 1) {
+      const page = await pdf.getPage(i);
+      const content = await page.getTextContent();
+
+      if (content.items.length === 0) {
+        continue;
+      }
+
+      let lastY;
+      const textItems = [];
+      for (const item of content.items) {
+        if ("str" in item) {
+          if (lastY === item.transform[5] || !lastY) {
+            textItems.push(item.str);
+          } else {
+            textItems.push(`\n${item.str}`);
+          }
+          lastY = item.transform[5];
+        }
+      }
+
+      const text = textItems.join("");
+      documents.push({
+        pageContent: text.trim(),
+        metadata: {
+          source: this.filePath,
+          pdf: {
+            version,
+            info: meta?.info,
+            metadata: meta?.metadata,
+            totalPages: pdf.numPages,
+          },
+          loc: { pageNumber: i },
+        },
+      });
+    }
+
+    if (this.splitPages) {
+      return documents;
+    }
+
+    if (documents.length === 0) {
+      return [];
+    }
+
+    return [
+      {
+        pageContent: documents.map((doc) => doc.pageContent).join("\n\n"),
+        metadata: {
+          source: this.filePath,
+          pdf: {
+            version,
+            info: meta?.info,
+            metadata: meta?.metadata,
+            totalPages: pdf.numPages,
+          },
+        },
+      },
+    ];
+  }
+
+  async getPdfJS() {
+    try {
+      const pdfjs = await import("pdf-parse/lib/pdf.js/v1.10.100/build/pdf.js");
+      return { getDocument: pdfjs.getDocument, version: pdfjs.version };
+    } catch (e) {
+      console.error(e);
+      throw new Error(
+        "Failed to load pdf-parse. Please install it with eg. `npm install pdf-parse`."
+      );
+    }
+  }
+}
+
+module.exports = PDFLoader;

package/collector/processSingleFile/convert/asPDF/index.js

@@ -0,0 +1,86 @@
+const { v4 } = require("uuid");
+const {
+  createdDate,
+  trashFile,
+  writeToServerDocuments,
+} = require("../../../utils/files");
+const { tokenizeString } = require("../../../utils/tokenizer");
+const { default: slugify } = require("slugify");
+const PDFLoader = require("./PDFLoader");
+const OCRLoader = require("../../../utils/OCRLoader");
+
+async function asPdf({
+  fullFilePath = "",
+  filename = "",
+  options = {},
+  metadata = {},
+}) {
+  const pdfLoader = new PDFLoader(fullFilePath, {
+    splitPages: true,
+  });
+
+  console.log(`-- Working ${filename} --`);
+  const pageContent = [];
+  let docs = await pdfLoader.load();
+
+  if (docs.length === 0) {
+    console.log(
+      `[asPDF] No text content found for ${filename}. Will attempt OCR parse.`
+    );
+    docs = await new OCRLoader({
+      targetLanguages: options?.ocr?.langList,
+    }).ocrPDF(fullFilePath);
+  }
+
+  for (const doc of docs) {
+    console.log(
+      `-- Parsing content from pg ${
+        doc.metadata?.loc?.pageNumber || "unknown"
+      } --`
+    );
+    if (!doc.pageContent || !doc.pageContent.length) continue;
+    pageContent.push(doc.pageContent);
+  }
+
+  if (!pageContent.length) {
+    console.error(`[asPDF] Resulting text content was empty for ${filename}.`);
+    if (!options.absolutePath) trashFile(fullFilePath);
+    return {
+      success: false,
+      reason: `No text content found in ${filename}.`,
+      documents: [],
+    };
+  }
+
+  const content = pageContent.join("");
+  const data = {
+    id: v4(),
+    url: "file://" + fullFilePath,
+    title: metadata.title || filename,
+    docAuthor:
+      metadata.docAuthor ||
+      docs[0]?.metadata?.pdf?.info?.Creator ||
+      "no author found",
+    description:
+      metadata.description ||
+      docs[0]?.metadata?.pdf?.info?.Title ||
+      "No description found.",
+    docSource: metadata.docSource || "pdf file uploaded by the user.",
+    chunkSource: metadata.chunkSource || "",
+    published: createdDate(fullFilePath),
+    wordCount: content.split(" ").length,
+    pageContent: content,
+    token_count_estimate: tokenizeString(content),
+  };
+
+  const document = writeToServerDocuments({
+    data,
+    filename: `${slugify(filename)}-${data.id}`,
+    options: { parseOnly: options.parseOnly },
+  });
+  if (!options.absolutePath) trashFile(fullFilePath);
+  console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
+  return { success: true, reason: null, documents: [document] };
+}
+
+module.exports = asPdf;

package/collector/processSingleFile/convert/asTxt.js

@@ -0,0 +1,59 @@
+const { v4 } = require("uuid");
+const fs = require("fs");
+const { tokenizeString } = require("../../utils/tokenizer");
+const {
+  createdDate,
+  trashFile,
+  writeToServerDocuments,
+} = require("../../utils/files");
+const { default: slugify } = require("slugify");
+
+async function asTxt({
+  fullFilePath = "",
+  filename = "",
+  options = {},
+  metadata = {},
+}) {
+  let content = "";
+  try {
+    content = fs.readFileSync(fullFilePath, "utf8");
+  } catch (err) {
+    console.error("Could not read file!", err);
+  }
+
+  if (!content?.length) {
+    console.error(`Resulting text content was empty for ${filename}.`);
+    if (!options.absolutePath) trashFile(fullFilePath);
+    return {
+      success: false,
+      reason: `No text content found in ${filename}.`,
+      documents: [],
+    };
+  }
+
+  console.log(`-- Working ${filename} --`);
+  const data = {
+    id: v4(),
+    url: "file://" + fullFilePath,
+    title: metadata.title || filename,
+    docAuthor: metadata.docAuthor || "Unknown",
+    description: metadata.description || "Unknown",
+    docSource: metadata.docSource || "a text file uploaded by the user.",
+    chunkSource: metadata.chunkSource || "",
+    published: createdDate(fullFilePath),
+    wordCount: content.split(" ").length,
+    pageContent: content,
+    token_count_estimate: tokenizeString(content),
+  };
+
+  const document = writeToServerDocuments({
+    data,
+    filename: `${slugify(filename)}-${data.id}`,
+    options: { parseOnly: options.parseOnly },
+  });
+  if (!options.absolutePath) trashFile(fullFilePath);
+  console.log(`[SUCCESS]: ${filename} converted & ready for embedding.\n`);
+  return { success: true, reason: null, documents: [document] };
+}
+
+module.exports = asTxt;

package/collector/processSingleFile/convert/asXlsx.js

@@ -0,0 +1,193 @@
+const { v4 } = require("uuid");
+const xlsx = require("node-xlsx").default;
+const path = require("path");
+const fs = require("fs");
+const {
+  createdDate,
+  trashFile,
+  writeToServerDocuments,
+  documentsFolder,
+} = require("../../utils/files");
+const { tokenizeString } = require("../../utils/tokenizer");
+const { default: slugify } = require("slugify");
+
+function convertToCSV(data) {
+  return data
+    .map((row) =>
+      row
+        .map((cell) => {
+          if (cell === null || cell === undefined) return "";
+          if (typeof cell === "string" && cell.includes(","))
+            return `"${cell}"`;
+          return cell;
+        })
+        .join(",")
+    )
+    .join("\n");
+}
+
+async function asXlsx({
+  fullFilePath = "",
+  filename = "",
+  options = {},
+  metadata = {},
+}) {
+  const documents = [];
+
+  try {
+    const workSheetsFromFile = xlsx.parse(fullFilePath);
+
+    if (options.parseOnly) {
+      const allSheetContents = [];
+      let totalWordCount = 0;
+      const sheetNames = [];
+
+      for (const sheet of workSheetsFromFile) {
+        const processed = processSheet(sheet);
+        if (!processed) continue;
+
+        const { name, content, wordCount } = processed;
+        sheetNames.push(name);
+        allSheetContents.push(`\nSheet: ${name}\n${content}`);
+        totalWordCount += wordCount;
+      }
+
+      if (allSheetContents.length === 0) {
+        console.log(`No valid sheets found in ${filename}.`);
+        return {
+          success: false,
+          reason: `No valid sheets found in ${filename}.`,
+          documents: [],
+        };
+      }
+
+      const combinedContent = allSheetContents.join("\n");
+      const sheetListText =
+        sheetNames.length > 1
+          ? ` (Sheets: ${sheetNames.join(", ")})`
+          : ` (Sheet: ${sheetNames[0]})`;
+
+      const combinedData = {
+        id: v4(),
+        url: `file://${fullFilePath}`,
+        title: metadata.title || `${filename}${sheetListText}`,
+        docAuthor: metadata.docAuthor || "Unknown",
+        description:
+          metadata.description ||
+          `Spreadsheet data from ${filename} containing ${sheetNames.length} ${
+            sheetNames.length === 1 ? "sheet" : "sheets"
+          }`,
+        docSource: metadata.docSource || "an xlsx file uploaded by the user.",
+        chunkSource: metadata.chunkSource || "",
+        published: createdDate(fullFilePath),
+        wordCount: totalWordCount,
+        pageContent: combinedContent,
+        token_count_estimate: tokenizeString(combinedContent),
+      };
+
+      const document = writeToServerDocuments({
+        data: combinedData,
+        filename: `${slugify(path.basename(filename))}-${combinedData.id}`,
+        destinationOverride: null,
+        options: { parseOnly: true },
+      });
+      documents.push(document);
+      console.log(`[SUCCESS]: ${filename} converted & ready for embedding.`);
+    } else {
+      const folderName = slugify(
+        `${path.basename(filename)}-${v4().slice(0, 4)}`,
+        {
+          lower: true,
+          trim: true,
+        }
+      );
+      const outFolderPath = path.resolve(documentsFolder, folderName);
+      if (!fs.existsSync(outFolderPath))
+        fs.mkdirSync(outFolderPath, { recursive: true });
+
+      for (const sheet of workSheetsFromFile) {
+        const processed = processSheet(sheet);
+        if (!processed) continue;
+
+        const { name, content, wordCount } = processed;
+        const sheetData = {
+          id: v4(),
+          url: `file://${path.join(outFolderPath, `${slugify(name)}.csv`)}`,
+          title: metadata.title || `${filename} - Sheet:${name}`,
+          docAuthor: metadata.docAuthor || "Unknown",
+          description:
+            metadata.description || `Spreadsheet data from sheet: ${name}`,
+          docSource: metadata.docSource || "an xlsx file uploaded by the user.",
+          chunkSource: metadata.chunkSource || "",
+          published: createdDate(fullFilePath),
+          wordCount: wordCount,
+          pageContent: content,
+          token_count_estimate: tokenizeString(content),
+        };
+
+        const document = writeToServerDocuments({
+          data: sheetData,
+          filename: `sheet-${slugify(name)}`,
+          destinationOverride: outFolderPath,
+          options: { parseOnly: options.parseOnly },
+        });
+        documents.push(document);
+        console.log(
+          `[SUCCESS]: Sheet "${name}" converted & ready for embedding.`
+        );
+      }
+    }
+  } catch (err) {
+    console.error("Could not process xlsx file!", err);
+    return {
+      success: false,
+      reason: `Error processing ${filename}: ${err.message}`,
+      documents: [],
+    };
+  } finally {
+    if (!options.absolutePath) trashFile(fullFilePath);
+  }
+
+  if (documents.length === 0) {
+    console.error(`No valid sheets found in ${filename}.`);
+    return {
+      success: false,
+      reason: `No valid sheets found in ${filename}.`,
+      documents: [],
+    };
+  }
+
+  console.log(
+    `[SUCCESS]: ${filename} fully processed. Created ${documents.length} document(s).\n`
+  );
+  return { success: true, reason: null, documents };
+}
+
+/**
+ * Processes a single sheet and returns its content and metadata
+ * @param {{name: string, data: Array<Array<string|number|null|undefined>>}} sheet - Parsed sheet with name and 2D array of cell values
+ * @returns {{name: string, content: string, wordCount: number}|null} - Object with name, CSV content, and word count, or null if sheet is empty
+ */
+function processSheet(sheet) {
+  try {
+    const { name, data } = sheet;
+    const content = convertToCSV(data);
+
+    if (!content?.length) {
+      console.log(`Sheet "${name}" is empty. Skipping.`);
+      return null;
+    }
+
+    console.log(`-- Processing sheet: ${name} --`);
+    return {
+      name,
+      content,
+      wordCount: content.split(/\s+/).length,
+    };
+  } catch (err) {
+    console.error(`Error processing sheet "${sheet.name}":`, err);
+    return null;
+  }
+}
+
+module.exports = asXlsx;

package/collector/processSingleFile/index.js

@@ -0,0 +1,95 @@
+const path = require("path");
+const fs = require("fs");
+const {
+  WATCH_DIRECTORY,
+  SUPPORTED_FILETYPE_CONVERTERS,
+} = require("../utils/constants");
+const {
+  trashFile,
+  isTextType,
+  normalizePath,
+  isWithin,
+} = require("../utils/files");
+const RESERVED_FILES = ["__HOTDIR__.md"];
+
+/**
+ * Process a single file and return the documents
+ * @param {string} targetFilename - The filename to process
+ * @param {Object} options - The options for the file processing
+ * @param {boolean} options.parseOnly - If true, the file will not be saved as a document even when `writeToServerDocuments` is called in the handler. Must be explicitly set to true to use.
+ * @param {string} options.absolutePath - If provided, use this absolute path instead of resolving relative to WATCH_DIRECTORY. For internal use only.
+ * @param {Object} metadata - The metadata for the file processing
+ * @returns {Promise<{success: boolean, reason: string, documents: Object[]}>} - The documents from the file processing
+ */
+async function processSingleFile(targetFilename, options = {}, metadata = {}) {
+  const fullFilePath = normalizePath(
+    options.absolutePath || path.resolve(WATCH_DIRECTORY, targetFilename)
+  );
+
+  // If absolute path is not provided, check if the file is within the watch directory
+  // to prevent unauthorized paths from being processed.
+  if (
+    !options.absolutePath &&
+    !isWithin(path.resolve(WATCH_DIRECTORY), fullFilePath)
+  )
+    return {
+      success: false,
+      reason: "Filename is a not a valid path to process.",
+      documents: [],
+    };
+
+  if (RESERVED_FILES.includes(targetFilename))
+    return {
+      success: false,
+      reason: "Filename is a reserved filename and cannot be processed.",
+      documents: [],
+    };
+
+  if (!fs.existsSync(fullFilePath))
+    return {
+      success: false,
+      reason: "File does not exist in upload directory.",
+      documents: [],
+    };
+
+  const fileExtension = path.extname(fullFilePath).toLowerCase();
+  if (fullFilePath.includes(".") && !fileExtension) {
+    return {
+      success: false,
+      reason: `No file extension found. This file cannot be processed.`,
+      documents: [],
+    };
+  }
+
+  let processFileAs = fileExtension;
+  if (!SUPPORTED_FILETYPE_CONVERTERS.hasOwnProperty(fileExtension)) {
+    if (isTextType(fullFilePath)) {
+      console.log(
+        `\x1b[33m[Collector]\x1b[0m The provided filetype of ${fileExtension} does not have a preset and will be processed as .txt.`
+      );
+      processFileAs = ".txt";
+    } else {
+      // If absolute path is provided, do NOT trash the file since it is a user provided path.
+      if (!options.absolutePath) trashFile(fullFilePath);
+      return {
+        success: false,
+        reason: `File extension ${fileExtension} not supported for parsing and cannot be assumed as text file type.`,
+        documents: [],
+      };
+    }
+  }
+
+  const FileTypeProcessor = require(SUPPORTED_FILETYPE_CONVERTERS[
+    processFileAs
+  ]);
+  return await FileTypeProcessor({
+    fullFilePath,
+    filename: targetFilename,
+    options,
+    metadata,
+  });
+}
+
+module.exports = {
+  processSingleFile,
+};

package/collector/utils/EncryptionWorker/index.js

@@ -0,0 +1,77 @@
+const crypto = require("crypto");
+
+// Differs from EncryptionManager in that is does not set or define the keys that will be used
+// to encrypt or read data and it must be told the key (as base64 string) explicitly that will be used and is provided to
+// the class on creation. This key should be the same `key` that is used by the EncryptionManager class.
+class EncryptionWorker {
+  constructor(presetKeyBase64 = "") {
+    this.key = Buffer.from(presetKeyBase64, "base64");
+    this.algorithm = "aes-256-cbc";
+    this.separator = ":";
+  }
+
+  log(text, ...args) {
+    console.log(`\x1b[36m[EncryptionManager]\x1b[0m ${text}`, ...args);
+  }
+
+  /**
+   * Give a chunk source, parse its payload query param and expand that object back into the URL
+   * as additional query params
+   * @param {string} chunkSource
+   * @returns {URL} Javascript URL object with query params decrypted from payload query param.
+   */
+  expandPayload(chunkSource = "") {
+    try {
+      const url = new URL(chunkSource);
+      if (!url.searchParams.has("payload")) return url;
+
+      const decryptedPayload = this.decrypt(url.searchParams.get("payload"));
+      const encodedParams = JSON.parse(decryptedPayload);
+      url.searchParams.delete("payload"); // remove payload prop
+
+      // Add all query params needed to replay as query params
+      Object.entries(encodedParams).forEach(([key, value]) =>
+        url.searchParams.append(key, value)
+      );
+      return url;
+    } catch (e) {
+      console.error(e);
+    }
+    return new URL(chunkSource);
+  }
+
+  encrypt(plainTextString = null) {
+    try {
+      if (!plainTextString)
+        throw new Error("Empty string is not valid for this method.");
+      const iv = crypto.randomBytes(16);
+      const cipher = crypto.createCipheriv(this.algorithm, this.key, iv);
+      const encrypted = cipher.update(plainTextString, "utf8", "hex");
+      return [
+        encrypted + cipher.final("hex"),
+        Buffer.from(iv).toString("hex"),
+      ].join(this.separator);
+    } catch (e) {
+      this.log(e);
+      return null;
+    }
+  }
+
+  decrypt(encryptedString) {
+    try {
+      const [encrypted, iv] = encryptedString.split(this.separator);
+      if (!iv) throw new Error("IV not found");
+      const decipher = crypto.createDecipheriv(
+        this.algorithm,
+        this.key,
+        Buffer.from(iv, "hex")
+      );
+      return decipher.update(encrypted, "hex", "utf8") + decipher.final("utf8");
+    } catch (e) {
+      this.log(e);
+      return null;
+    }
+  }
+}
+
+module.exports = { EncryptionWorker };