totem-llm 0.1.0

package/collector/utils/extensions/RepoLoader/index.js

@@ -0,0 +1,41 @@
+/**
+ * Dynamically load the correct repository loader from a specific platform
+ * by default will return GitHub.
+ * @param {('github'|'gitlab')} platform
+ * @returns {import("./GithubRepo/RepoLoader")|import("./GitlabRepo/RepoLoader")} the repo loader class for provider
+ */
+function resolveRepoLoader(platform = "github") {
+  switch (platform) {
+    case "github":
+      console.log(`Loading GitHub RepoLoader...`);
+      return require("./GithubRepo/RepoLoader");
+    case "gitlab":
+      console.log(`Loading GitLab RepoLoader...`);
+      return require("./GitlabRepo/RepoLoader");
+    default:
+      console.log(`Loading GitHub RepoLoader...`);
+      return require("./GithubRepo/RepoLoader");
+  }
+}
+
+/**
+ * Dynamically load the correct repository loader function from a specific platform
+ * by default will return Github.
+ * @param {('github'|'gitlab')} platform
+ * @returns {import("./GithubRepo")['fetchGithubFile'] | import("./GitlabRepo")['fetchGitlabFile']} the repo loader class for provider
+ */
+function resolveRepoLoaderFunction(platform = "github") {
+  switch (platform) {
+    case "github":
+      console.log(`Loading GitHub loader function...`);
+      return require("./GithubRepo").loadGithubRepo;
+    case "gitlab":
+      console.log(`Loading GitLab loader function...`);
+      return require("./GitlabRepo").loadGitlabRepo;
+    default:
+      console.log(`Loading GitHub loader function...`);
+      return require("./GithubRepo").loadGithubRepo;
+  }
+}
+
+module.exports = { resolveRepoLoader, resolveRepoLoaderFunction };

package/collector/utils/extensions/WebsiteDepth/index.js

@@ -0,0 +1,211 @@
+const { v4 } = require("uuid");
+const {
+  PuppeteerWebBaseLoader,
+} = require("langchain/document_loaders/web/puppeteer");
+const { default: slugify } = require("slugify");
+const { parse } = require("node-html-parser");
+const { writeToServerDocuments, documentsFolder } = require("../../files");
+const { tokenizeString } = require("../../tokenizer");
+const path = require("path");
+const fs = require("fs");
+const RuntimeSettings = require("../../runtimeSettings");
+
+async function discoverLinks(startUrl, maxDepth = 1, maxLinks = 20) {
+  const baseUrl = new URL(startUrl);
+  const discoveredLinks = new Set([startUrl]);
+  let queue = [[startUrl, 0]]; // [url, currentDepth]
+  const scrapedUrls = new Set();
+
+  for (let currentDepth = 0; currentDepth < maxDepth; currentDepth++) {
+    const levelSize = queue.length;
+    const nextQueue = [];
+
+    for (let i = 0; i < levelSize && discoveredLinks.size < maxLinks; i++) {
+      const [currentUrl, urlDepth] = queue[i];
+
+      if (!scrapedUrls.has(currentUrl)) {
+        scrapedUrls.add(currentUrl);
+        const newLinks = await getPageLinks(currentUrl, baseUrl);
+
+        for (const link of newLinks) {
+          if (!discoveredLinks.has(link) && discoveredLinks.size < maxLinks) {
+            discoveredLinks.add(link);
+            if (urlDepth + 1 < maxDepth) {
+              nextQueue.push([link, urlDepth + 1]);
+            }
+          }
+        }
+      }
+    }
+
+    queue = nextQueue;
+    if (queue.length === 0 || discoveredLinks.size >= maxLinks) break;
+  }
+
+  return Array.from(discoveredLinks);
+}
+
+async function getPageLinks(url, baseUrl) {
+  try {
+    const runtimeSettings = new RuntimeSettings();
+    /** @type {import('puppeteer').PuppeteerLaunchOptions} */
+    let launchConfig = { headless: "new" };
+
+    /* On MacOS 15.1, the headless=new option causes the browser to crash immediately.
+     * It is not clear why this is the case, but it is reproducible. Since AnythinglLM
+     * in production runs in a container, we can disable headless mode to workaround the issue for development purposes.
+     *
+     * This may show a popup window when scraping a page in development mode.
+     * This is expected behavior if seen in development mode on MacOS 15+
+     */
+    if (
+      process.platform === "darwin" &&
+      process.env.NODE_ENV === "development"
+    ) {
+      console.log(
+        "Darwin Development Mode: Disabling headless mode to prevent Chromium from crashing."
+      );
+      launchConfig.headless = "false";
+    }
+
+    const loader = new PuppeteerWebBaseLoader(url, {
+      launchOptions: {
+        headless: launchConfig.headless,
+        ignoreHTTPSErrors: true,
+        args: runtimeSettings.get("browserLaunchArgs"),
+      },
+      gotoOptions: { waitUntil: "networkidle2" },
+    });
+    const docs = await loader.load();
+    const html = docs[0].pageContent;
+    const links = extractLinks(html, baseUrl);
+    return links;
+  } catch (error) {
+    console.error(`Failed to get page links from ${url}.`, error);
+    return [];
+  }
+}
+
+function extractLinks(html, baseUrl) {
+  const root = parse(html);
+  const links = root.querySelectorAll("a");
+  const extractedLinks = new Set();
+
+  for (const link of links) {
+    const href = link.getAttribute("href");
+    if (href) {
+      const absoluteUrl = new URL(href, baseUrl.href).href;
+      if (
+        absoluteUrl.startsWith(
+          baseUrl.origin + baseUrl.pathname.split("/").slice(0, -1).join("/")
+        )
+      ) {
+        extractedLinks.add(absoluteUrl);
+      }
+    }
+  }
+
+  return Array.from(extractedLinks);
+}
+
+async function bulkScrapePages(links, outFolderPath) {
+  const runtimeSettings = new RuntimeSettings();
+  /** @type {import('puppeteer').PuppeteerLaunchOptions} */
+  let launchConfig = { headless: "new" };
+
+  /* On MacOS 15.1, the headless=new option causes the browser to crash immediately.
+   * It is not clear why this is the case, but it is reproducible. Since AnythinglLM
+   * in production runs in a container, we can disable headless mode to workaround the issue for development purposes.
+   *
+   * This may show a popup window when scraping a page in development mode.
+   * This is expected behavior if seen in development mode on MacOS 15+
+   */
+  if (process.platform === "darwin" && process.env.NODE_ENV === "development") {
+    console.log(
+      "Darwin Development Mode: Disabling headless mode to prevent Chromium from crashing."
+    );
+    launchConfig.headless = "false";
+  }
+
+  const scrapedData = [];
+
+  for (let i = 0; i < links.length; i++) {
+    const link = links[i];
+    console.log(`Scraping ${i + 1}/${links.length}: ${link}`);
+
+    try {
+      const loader = new PuppeteerWebBaseLoader(link, {
+        launchOptions: {
+          headless: launchConfig.headless,
+          ignoreHTTPSErrors: true,
+          args: runtimeSettings.get("browserLaunchArgs"),
+        },
+        gotoOptions: { waitUntil: "networkidle2" },
+        async evaluate(page, browser) {
+          const result = await page.evaluate(() => document.body.innerText);
+          await browser.close();
+          return result;
+        },
+      });
+      const docs = await loader.load();
+      const content = docs[0].pageContent;
+
+      if (!content.length) {
+        console.warn(`Empty content for ${link}. Skipping.`);
+        continue;
+      }
+
+      const url = new URL(link);
+      const decodedPathname = decodeURIComponent(url.pathname);
+      const filename = `${url.hostname}${decodedPathname.replace(/\//g, "_")}`;
+
+      const data = {
+        id: v4(),
+        url: "file://" + slugify(filename) + ".html",
+        title: slugify(filename) + ".html",
+        docAuthor: "no author found",
+        description: "No description found.",
+        docSource: "URL link uploaded by the user.",
+        chunkSource: `link://${link}`,
+        published: new Date().toLocaleString(),
+        wordCount: content.split(" ").length,
+        pageContent: content,
+        token_count_estimate: tokenizeString(content),
+      };
+
+      writeToServerDocuments({
+        data,
+        filename: data.title,
+        destinationOverride: outFolderPath,
+      });
+      scrapedData.push(data);
+
+      console.log(`Successfully scraped ${link}.`);
+    } catch (error) {
+      console.error(`Failed to scrape ${link}.`, error);
+    }
+  }
+
+  return scrapedData;
+}
+
+async function websiteScraper(startUrl, depth = 1, maxLinks = 20) {
+  const websiteName = new URL(startUrl).hostname;
+  const outFolder = slugify(
+    `${slugify(websiteName)}-${v4().slice(0, 4)}`
+  ).toLowerCase();
+  const outFolderPath = path.resolve(documentsFolder, outFolder);
+  console.log("Discovering links...");
+  const linksToScrape = await discoverLinks(startUrl, depth, maxLinks);
+  console.log(`Found ${linksToScrape.length} links to scrape.`);
+
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
+  console.log("Starting bulk scraping...");
+  const scrapedData = await bulkScrapePages(linksToScrape, outFolderPath);
+  console.log(`Scraped ${scrapedData.length} pages.`);
+
+  return scrapedData;
+}
+
+module.exports = websiteScraper;

package/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/index.js

@@ -0,0 +1,99 @@
+const { validYoutubeVideoUrl } = require("../../../url");
+
+/*
+ * This is just a custom implementation of the Langchain JS YouTubeLoader class
+ * as the dependency for YoutubeTranscript is quite fickle and its a rat race to keep it up
+ * and instead of waiting for patches we can just bring this simple script in-house and at least
+ * be able to patch it since its so flaky. When we have more connectors we can kill this because
+ * it will be a pain to maintain over time.
+ */
+class YoutubeLoader {
+  #videoId;
+  #language;
+  #addVideoInfo;
+
+  constructor({ videoId = null, language = null, addVideoInfo = false } = {}) {
+    if (!videoId) throw new Error("Invalid video id!");
+    this.#videoId = videoId;
+    this.#language = language;
+    this.#addVideoInfo = addVideoInfo;
+  }
+
+  /**
+   * Extracts the videoId from a YouTube video URL.
+   * @param url The URL of the YouTube video.
+   * @returns The videoId of the YouTube video.
+   */
+  static getVideoID(url) {
+    const videoId = validYoutubeVideoUrl(url, true);
+    if (videoId) return videoId;
+    throw new Error("Failed to get youtube video id from the url");
+  }
+
+  /**
+   * Creates a new instance of the YoutubeLoader class from a YouTube video
+   * URL.
+   * @param url The URL of the YouTube video.
+   * @param config Optional configuration options for the YoutubeLoader instance, excluding the videoId.
+   * @returns A new instance of the YoutubeLoader class.
+   */
+  static createFromUrl(url, config = {}) {
+    const videoId = YoutubeLoader.getVideoID(url);
+    return new YoutubeLoader({ ...config, videoId });
+  }
+
+  /**
+   * Loads the transcript and video metadata from the specified YouTube
+   * video. It uses the youtube-transcript library to fetch the transcript
+   * and the youtubei.js library to fetch the video metadata.
+   * @returns Langchain like doc that is 1 element with PageContent and
+   */
+  async load() {
+    let transcript;
+    const metadata = {
+      source: this.#videoId,
+    };
+    try {
+      const fetchTranscript = await import("youtube-transcript-plus").then(
+        (module) => module.fetchTranscript
+      );
+      const transcriptSegments = await fetchTranscript(this.#videoId, {
+        lang: this.#language,
+      });
+      if (!transcriptSegments || transcriptSegments.length === 0)
+        throw new Error("Transcription not found");
+      transcript = this.#convertTranscriptSegmentsToText(transcriptSegments);
+      if (this.#addVideoInfo) {
+        const { Innertube } = require("youtubei.js");
+        const youtube = await Innertube.create();
+        const info = (await youtube.getBasicInfo(this.#videoId)).basic_info;
+        metadata.description = info.short_description;
+        metadata.title = info.title;
+        metadata.view_count = info.view_count;
+        metadata.author = info.author;
+      }
+    } catch (e) {
+      throw new Error(
+        `Failed to get YouTube video transcription: ${e?.message}`
+      );
+    }
+    return [
+      {
+        pageContent: transcript,
+        metadata,
+      },
+    ];
+  }
+
+  #convertTranscriptSegmentsToText(transcriptSegments) {
+    return transcriptSegments
+      .map((segment) =>
+        typeof segment === "string" ? segment : segment.text || ""
+      )
+      .join(" ")
+      .replace(/\s+/g, " ")
+      .trim();
+  }
+}
+
+module.exports.YoutubeLoader = YoutubeLoader;

package/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js

@@ -0,0 +1,246 @@
+const { validYoutubeVideoUrl } = require("../../../url");
+
+class YoutubeTranscriptError extends Error {
+  constructor(message) {
+    super(`[YoutubeTranscript] ${message}`);
+  }
+}
+
+/**
+ * Handles fetching and parsing YouTube video transcripts
+ */
+class YoutubeTranscript {
+  /**
+   * Encodes a string as a protobuf field
+   * @param {number} fieldNumber - The protobuf field number
+   * @param {string} str - The string to encode
+   * @returns {Buffer} Encoded protobuf field
+   */
+  static #encodeProtobufString(fieldNumber, str) {
+    const utf8Bytes = Buffer.from(str, "utf8");
+    const tag = (fieldNumber << 3) | 2; // wire type 2 for string
+    const lengthBytes = this.#encodeVarint(utf8Bytes.length);
+
+    return Buffer.concat([
+      Buffer.from([tag]),
+      Buffer.from(lengthBytes),
+      utf8Bytes,
+    ]);
+  }
+
+  /**
+   * Encodes a number as a protobuf varint
+   * @param {number} value - The number to encode
+   * @returns {number[]} Encoded varint bytes
+   */
+  static #encodeVarint(value) {
+    const bytes = [];
+    while (value >= 0x80) {
+      bytes.push((value & 0x7f) | 0x80);
+      value >>>= 7;
+    }
+    bytes.push(value);
+    return bytes;
+  }
+
+  /**
+   * Creates a base64 encoded protobuf message
+   * @param {Object} param - The parameters to encode
+   * @param {string} param.param1 - First parameter
+   * @param {string} param.param2 - Second parameter
+   * @returns {string} Base64 encoded protobuf
+   */
+  static #getBase64Protobuf({ param1, param2 }) {
+    const field1 = this.#encodeProtobufString(1, param1);
+    const field2 = this.#encodeProtobufString(2, param2);
+    return Buffer.concat([field1, field2]).toString("base64");
+  }
+
+  /**
+   * Extracts transcript text from YouTube API response
+   * @param {Object} responseData - The YouTube API response
+   * @returns {string} Combined transcript text
+   */
+  static #extractTranscriptFromResponse(responseData) {
+    const transcriptRenderer =
+      responseData.actions?.[0]?.updateEngagementPanelAction?.content
+        ?.transcriptRenderer;
+    if (!transcriptRenderer) {
+      throw new Error("No transcript data found in response");
+    }
+
+    const segments =
+      transcriptRenderer.content?.transcriptSearchPanelRenderer?.body
+        ?.transcriptSegmentListRenderer?.initialSegments;
+    if (!segments) {
+      throw new Error("Transcript segments not found in response");
+    }
+
+    return segments
+      .map((segment) => {
+        const runs = segment.transcriptSegmentRenderer?.snippet?.runs;
+        return runs ? runs.map((run) => run.text).join("") : "";
+      })
+      .filter((text) => text)
+      .join(" ")
+      .trim()
+      .replace(/\s+/g, " ");
+  }
+
+  /**
+   * Calculates a preference score for a caption track to determine the best match
+   * @param {Object} track - The caption track object from YouTube
+   * @param {string} track.languageCode - ISO language code (e.g., 'zh-HK', 'en', 'es')
+   * @param {string} track.kind - Track type ('asr' for auto-generated, "" for human-transcribed)
+   * @param {string[]} preferredLanguages - Array of language codes in preference order (e.g., ['zh-HK', 'en'])
+   * @returns {number} Preference score (lower is better)
+   */
+  static #calculatePreferenceScore(track, preferredLanguages) {
+    // Language preference: index in preferredLanguages array (0 = most preferred)
+    const languagePreference = preferredLanguages.indexOf(track.languageCode);
+    const languageScore = languagePreference === -1 ? 9999 : languagePreference;
+
+    // Kind bonus: prefer human-transcribed (undefined) over auto-generated ('asr')
+    const kindBonus = track.kind === "asr" ? 0.5 : 0;
+
+    return languageScore + kindBonus;
+  }
+
+  /**
+   * Finds the most suitable caption track based on preferred languages
+   * @param {string} videoBody - The raw HTML response from YouTube
+   * @param {string[]} preferredLanguages - Array of language codes in preference order
+   * @returns {Object|null} The selected caption track or null if none found
+   */
+  static #findPreferredCaptionTrack(videoBody, preferredLanguages) {
+    const captionsConfigJson = videoBody.match(
+      /"captions":(.*?),"videoDetails":/s
+    );
+
+    const captionsConfig = captionsConfigJson?.[1]
+      ? JSON.parse(captionsConfigJson[1])
+      : null;
+
+    const captionTracks = captionsConfig
+      ? captionsConfig.playerCaptionsTracklistRenderer.captionTracks
+      : null;
+
+    if (!captionTracks || captionTracks.length === 0) {
+      return null;
+    }
+
+    const sortedTracks = [...captionTracks].sort((a, b) => {
+      const scoreA = this.#calculatePreferenceScore(a, preferredLanguages);
+      const scoreB = this.#calculatePreferenceScore(b, preferredLanguages);
+      return scoreA - scoreB;
+    });
+
+    return sortedTracks[0];
+  }
+
+  /**
+   * Fetches video page content and finds the preferred caption track
+   * @param {string} videoId - YouTube video ID
+   * @param {string[]} preferredLanguages - Array of preferred language codes
+   * @returns {Promise<Object>} The preferred caption track
+   * @throws {YoutubeTranscriptError} If no suitable caption track is found
+   */
+  static async #getPreferredCaptionTrack(videoId, preferredLanguages) {
+    const videoResponse = await fetch(
+      `https://www.youtube.com/watch?v=${videoId}`,
+      { credentials: "omit" }
+    );
+    const videoBody = await videoResponse.text();
+
+    const preferredCaptionTrack = this.#findPreferredCaptionTrack(
+      videoBody,
+      preferredLanguages
+    );
+
+    if (!preferredCaptionTrack) {
+      throw new YoutubeTranscriptError(
+        "No suitable caption track found for the video"
+      );
+    }
+
+    return preferredCaptionTrack;
+  }
+
+  /**
+   * Fetch transcript from YouTube video
+   * @param {string} videoId - Video URL or video identifier
+   * @param {Object} config - Configuration options
+   * @param {string} [config.lang='en'] - Language code (e.g., 'en', 'es', 'fr')
+   * @returns {Promise<string>} Video transcript text
+   */
+  static async fetchTranscript(videoId, config = {}) {
+    const preferredLanguages = config?.lang ? [config?.lang, "en"] : ["en"];
+    const identifier = this.retrieveVideoId(videoId);
+
+    try {
+      const preferredCaptionTrack = await this.#getPreferredCaptionTrack(
+        identifier,
+        preferredLanguages
+      );
+
+      const innerProto = this.#getBase64Protobuf({
+        param1: preferredCaptionTrack.kind || "",
+        param2: preferredCaptionTrack.languageCode,
+      });
+
+      const params = this.#getBase64Protobuf({
+        param1: identifier,
+        param2: innerProto,
+      });
+
+      const response = await fetch(
+        "https://www.youtube.com/youtubei/v1/get_transcript",
+        {
+          method: "POST",
+          headers: {
+            "Content-Type": "application/json",
+            "User-Agent":
+              "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/85.0.4183.83 Safari/537.36,gzip(gfe)",
+          },
+          body: JSON.stringify({
+            context: {
+              client: {
+                clientName: "WEB",
+                clientVersion: "2.20240826.01.00",
+              },
+            },
+            params,
+          }),
+        }
+      );
+
+      if (!response.ok) {
+        throw new Error(`HTTP error! status: ${response.status}`);
+      }
+
+      const responseData = await response.json();
+      return this.#extractTranscriptFromResponse(responseData);
+    } catch (e) {
+      throw new YoutubeTranscriptError(e.message || e);
+    }
+  }
+
+  /**
+   * Extract video ID from a YouTube URL or verify an existing ID
+   * @param {string} videoId - Video URL or ID
+   * @returns {string} YouTube video ID
+   */
+  static retrieveVideoId(videoId) {
+    if (videoId.length === 11) return videoId; // already a valid ID most likely
+    const matchedId = validYoutubeVideoUrl(videoId, true);
+    if (matchedId) return matchedId;
+    throw new YoutubeTranscriptError(
+      "Impossible to retrieve Youtube video ID."
+    );
+  }
+}
+
+module.exports = {
+  YoutubeTranscript,
+  YoutubeTranscriptError,
+};