totem-llm 0.1.0

package/server/models/communityHub.js

@@ -0,0 +1,213 @@
+const ImportedPlugin = require("../utils/agents/imported");
+
+/**
+ * An interface to the AnythingLLM Community Hub external API.
+ */
+const CommunityHub = {
+  importPrefix: "allm-community-id",
+  apiBase:
+    process.env.NODE_ENV === "development"
+      ? "http://127.0.0.1:5001/anythingllm-hub/us-central1/external/v1"
+      : "https://hub.external.anythingllm.com/v1",
+  supportedStaticItemTypes: ["system-prompt", "agent-flow", "slash-command"],
+
+  /**
+   * Validate an import ID and return the entity type and ID.
+   * @param {string} importId - The import ID to validate.
+   * @returns {{entityType: string | null, entityId: string | null}}
+   */
+  validateImportId: function (importId) {
+    if (
+      !importId ||
+      !importId.startsWith(this.importPrefix) ||
+      importId.split(":").length !== 3
+    )
+      return { entityType: null, entityId: null };
+    const [_, entityType, entityId] = importId.split(":");
+    if (!entityType || !entityId) return { entityType: null, entityId: null };
+    return {
+      entityType: String(entityType).trim(),
+      entityId: String(entityId).trim(),
+    };
+  },
+
+  /**
+   * Fetch the explore items from the community hub that are publicly available.
+   * @returns {Promise<{agentSkills: {items: [], hasMore: boolean, totalCount: number}, systemPrompts: {items: [], hasMore: boolean, totalCount: number}, slashCommands: {items: [], hasMore: boolean, totalCount: number}}>}
+   */
+  fetchExploreItems: async function () {
+    return await fetch(`${this.apiBase}/explore`, {
+      method: "GET",
+    })
+      .then((response) => response.json())
+      .catch((error) => {
+        console.error("Error fetching explore items:", error);
+        return {
+          agentSkills: {
+            items: [],
+            hasMore: false,
+            totalCount: 0,
+          },
+          systemPrompts: {
+            items: [],
+            hasMore: false,
+            totalCount: 0,
+          },
+          slashCommands: {
+            items: [],
+            hasMore: false,
+            totalCount: 0,
+          },
+        };
+      });
+  },
+
+  /**
+   * Fetch a bundle item from the community hub.
+   * Bundle items are entities that require a downloadURL to be fetched from the community hub.
+   * so we can unzip and import them to the AnythingLLM instance.
+   * @param {string} importId - The import ID of the item.
+   * @returns {Promise<{url: string | null, item: object | null, error: string | null}>}
+   */
+  getBundleItem: async function (importId) {
+    const { entityType, entityId } = this.validateImportId(importId);
+    if (!entityType || !entityId)
+      return { item: null, error: "Invalid import ID" };
+
+    const { SystemSettings } = require("./systemSettings");
+    const { connectionKey } = await SystemSettings.hubSettings();
+    const { url, item, error } = await fetch(
+      `${this.apiBase}/${entityType}/${entityId}/pull`,
+      {
+        method: "GET",
+        headers: {
+          "Content-Type": "application/json",
+          ...(connectionKey
+            ? { Authorization: `Bearer ${connectionKey}` }
+            : {}),
+        },
+      }
+    )
+      .then((response) => response.json())
+      .catch((error) => {
+        console.error(
+          `Error fetching bundle item for import ID ${importId}:`,
+          error
+        );
+        return { url: null, item: null, error: error.message };
+      });
+    return { url, item, error };
+  },
+
+  /**
+   * Apply an item to the AnythingLLM instance. Used for simple items like slash commands and system prompts.
+   * @param {object} item - The item to apply.
+   * @param {object} options - Additional options for applying the item.
+   * @param {object|null} options.currentUser - The current user object.
+   * @returns {Promise<{success: boolean, error: string | null}>}
+   */
+  applyItem: async function (item, options = {}) {
+    if (!item) return { success: false, error: "Item is required" };
+
+    if (item.itemType === "system-prompt") {
+      if (!options?.workspaceSlug)
+        return { success: false, error: "Workspace slug is required" };
+
+      const { Workspace } = require("./workspace");
+      const workspace = await Workspace.get({
+        slug: String(options.workspaceSlug),
+      });
+      if (!workspace) return { success: false, error: "Workspace not found" };
+      await Workspace.update(workspace.id, { openAiPrompt: item.prompt });
+      return { success: true, error: null };
+    }
+
+    if (item.itemType === "slash-command") {
+      const { SlashCommandPresets } = require("./slashCommandsPresets");
+      await SlashCommandPresets.create(options?.currentUser?.id, {
+        command: SlashCommandPresets.formatCommand(String(item.command)),
+        prompt: String(item.prompt),
+        description: String(item.description),
+      });
+      return { success: true, error: null };
+    }
+
+    return {
+      success: false,
+      error: "Unsupported item type. Nothing to apply.",
+    };
+  },
+
+  /**
+   * Import a bundle item to the AnythingLLM instance by downloading the zip file and importing it.
+   * or whatever the item type requires.
+   * @param {{url: string, item: object}} params
+   * @returns {Promise<{success: boolean, error: string | null}>}
+   */
+  importBundleItem: async function ({ url, item }) {
+    if (item.itemType === "agent-skill") {
+      const { success, error } =
+        await ImportedPlugin.importCommunityItemFromUrl(url, item);
+      return { success, error };
+    }
+
+    return {
+      success: false,
+      error: "Unsupported item type. Nothing to import.",
+    };
+  },
+
+  fetchUserItems: async function (connectionKey) {
+    if (!connectionKey) return { createdByMe: {}, teamItems: [] };
+
+    return await fetch(`${this.apiBase}/items`, {
+      method: "GET",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${connectionKey}`,
+      },
+    })
+      .then((response) => response.json())
+      .catch((error) => {
+        console.error("Error fetching user items:", error);
+        return { createdByMe: {}, teamItems: [] };
+      });
+  },
+
+  /**
+   * Create a new item in the community hub - Only supports STATIC items for now.
+   * @param {string} itemType - The type of item to create
+   * @param {object} data - The item data
+   * @param {string} connectionKey - The hub connection key
+   * @returns {Promise<{success: boolean, error: string | null}>}
+   */
+  createStaticItem: async function (itemType, data, connectionKey) {
+    if (!connectionKey)
+      return { success: false, error: "Connection key is required" };
+    if (!this.supportedStaticItemTypes.includes(itemType))
+      return { success: false, error: "Unsupported item type" };
+
+    // If the item has special considerations or preprocessing, we can delegate that below before sending the request.
+    // eg: Agent flow files and such.
+
+    return await fetch(`${this.apiBase}/${itemType}/create`, {
+      method: "POST",
+      headers: {
+        "Content-Type": "application/json",
+        Authorization: `Bearer ${connectionKey}`,
+      },
+      body: JSON.stringify(data),
+    })
+      .then((response) => response.json())
+      .then((result) => {
+        if (!!result.error) throw new Error(result.error || "Unknown error");
+        return { success: true, error: null, itemId: result.item.id };
+      })
+      .catch((error) => {
+        console.error(`Error creating ${itemType}:`, error);
+        return { success: false, error: error.message };
+      });
+  },
+};
+
+module.exports = { CommunityHub };

package/server/models/documentSyncQueue.js

@@ -0,0 +1,252 @@
+const { BackgroundService } = require("../utils/BackgroundWorkers");
+const prisma = require("../utils/prisma");
+const { SystemSettings } = require("./systemSettings");
+
+/**
+ * @typedef {('link'|'youtube'|'confluence'|'github'|'gitlab')} validFileType
+ */
+
+const DocumentSyncQueue = {
+  featureKey: "experimental_live_file_sync",
+  // update the validFileTypes and .canWatch properties when adding elements here.
+  validFileTypes: [
+    "link",
+    "youtube",
+    "confluence",
+    "github",
+    "gitlab",
+    "drupalwiki",
+  ],
+  defaultStaleAfter: 604800000,
+  maxRepeatFailures: 5, // How many times a run can fail in a row before pruning.
+  writable: [],
+
+  bootWorkers: function () {
+    new BackgroundService().boot();
+  },
+
+  killWorkers: function () {
+    new BackgroundService().stop();
+  },
+
+  /** Check is the Document Sync/Watch feature is enabled and can be used. */
+  enabled: async function () {
+    return (
+      (await SystemSettings.get({ label: this.featureKey }))?.value ===
+      "enabled"
+    );
+  },
+
+  /**
+   * @param {import("@prisma/client").document_sync_queues} queueRecord - queue record to calculate for
+   */
+  calcNextSync: function (queueRecord) {
+    return new Date(Number(new Date()) + queueRecord.staleAfterMs);
+  },
+
+  /**
+   * Check if the document can be watched based on the metadata fields
+   * @param {object} metadata - metadata to check
+   * @param {string} metadata.title - title of the document
+   * @param {string} metadata.chunkSource - chunk source of the document
+   * @returns {boolean} - true if the document can be watched, false otherwise
+   */
+  canWatch: function ({ title, chunkSource = null } = {}) {
+    if (!chunkSource) return false;
+
+    if (chunkSource.startsWith("link://") && title.endsWith(".html"))
+      return true; // If is web-link material (prior to feature most chunkSources were links://)
+    if (chunkSource.startsWith("youtube://")) return true; // If is a youtube link
+    if (chunkSource.startsWith("confluence://")) return true; // If is a confluence document link
+    if (chunkSource.startsWith("github://")) return true; // If is a GitHub file reference
+    if (chunkSource.startsWith("gitlab://")) return true; // If is a GitLab file reference
+    if (chunkSource.startsWith("drupalwiki://")) return true; // If is a DrupalWiki document link
+    return false;
+  },
+
+  /**
+   * Creates Queue record and updates document watch status to true on Document record
+   * @param {import("@prisma/client").workspace_documents} document - document record to watch, must have `id`
+   */
+  watch: async function (document = null) {
+    if (!document) return false;
+    try {
+      const { Document } = require("./documents");
+
+      // Get all documents that are watched and share the same unique filename. If this value is
+      // non-zero then we exit early so that we do not have duplicated watch queues for the same file
+      // across many workspaces.
+      const workspaceDocIds = (
+        await Document.where({ filename: document.filename, watched: true })
+      ).map((rec) => rec.id);
+      const hasRecords =
+        (await this.count({ workspaceDocId: { in: workspaceDocIds } })) > 0;
+      if (hasRecords)
+        throw new Error(
+          `Cannot watch this document again - it already has a queue set.`
+        );
+
+      const queue = await prisma.document_sync_queues.create({
+        data: {
+          workspaceDocId: document.id,
+          nextSyncAt: new Date(Number(new Date()) + this.defaultStaleAfter),
+        },
+      });
+      await Document._updateAll(
+        { filename: document.filename },
+        { watched: true }
+      );
+      return queue || null;
+    } catch (error) {
+      console.error(error.message);
+      return null;
+    }
+  },
+
+  /**
+   * Deletes Queue record and updates document watch status to false on Document record
+   * @param {import("@prisma/client").workspace_documents} document - document record to unwatch, must have `id`
+   */
+  unwatch: async function (document = null) {
+    if (!document) return false;
+    try {
+      const { Document } = require("./documents");
+
+      // We could have been given a document to unwatch which is a clone of one that is already being watched but by another workspaceDocument id.
+      // so in this instance we need to delete any queues related to this document by any WorkspaceDocumentId it is referenced by.
+      const workspaceDocIds = (
+        await Document.where({ filename: document.filename, watched: true })
+      ).map((rec) => rec.id);
+      await this.delete({ workspaceDocId: { in: workspaceDocIds } });
+      await Document._updateAll(
+        { filename: document.filename },
+        { watched: false }
+      );
+      return true;
+    } catch (error) {
+      console.error(error.message);
+      return false;
+    }
+  },
+
+  _update: async function (id = null, data = {}) {
+    if (!id) throw new Error("No id provided for update");
+
+    try {
+      await prisma.document_sync_queues.update({
+        where: { id },
+        data,
+      });
+      return true;
+    } catch (error) {
+      console.error(error.message);
+      return false;
+    }
+  },
+
+  get: async function (clause = {}) {
+    try {
+      const queue = await prisma.document_sync_queues.findFirst({
+        where: clause,
+      });
+      return queue || null;
+    } catch (error) {
+      console.error(error.message);
+      return null;
+    }
+  },
+
+  where: async function (
+    clause = {},
+    limit = null,
+    orderBy = null,
+    include = {}
+  ) {
+    try {
+      const results = await prisma.document_sync_queues.findMany({
+        where: clause,
+        ...(limit !== null ? { take: limit } : {}),
+        ...(orderBy !== null ? { orderBy } : {}),
+        ...(include !== null ? { include } : {}),
+      });
+      return results;
+    } catch (error) {
+      console.error(error.message);
+      return [];
+    }
+  },
+
+  count: async function (clause = {}, limit = null) {
+    try {
+      const count = await prisma.document_sync_queues.count({
+        where: clause,
+        ...(limit !== null ? { take: limit } : {}),
+      });
+      return count;
+    } catch (error) {
+      console.error("FAILED TO COUNT DOCUMENTS.", error.message);
+      return 0;
+    }
+  },
+
+  delete: async function (clause = {}) {
+    try {
+      await prisma.document_sync_queues.deleteMany({ where: clause });
+      return true;
+    } catch (error) {
+      console.error(error.message);
+      return false;
+    }
+  },
+
+  /**
+   * Gets the "stale" queues where the queue's nextSyncAt is less than the current time
+   * @returns {Promise<(
+   *  import("@prisma/client").document_sync_queues &
+   * { workspaceDoc: import("@prisma/client").workspace_documents &
+   *  { workspace: import("@prisma/client").workspaces }
+   * })[]}>}
+   */
+  staleDocumentQueues: async function () {
+    const queues = await this.where(
+      {
+        nextSyncAt: {
+          lte: new Date().toISOString(),
+        },
+      },
+      null,
+      null,
+      {
+        workspaceDoc: {
+          include: {
+            workspace: true,
+          },
+        },
+      }
+    );
+    return queues;
+  },
+
+  saveRun: async function (queueId = null, status = null, result = {}) {
+    const { DocumentSyncRun } = require("./documentSyncRun");
+    return DocumentSyncRun.save(queueId, status, result);
+  },
+
+  /**
+   * Updates document to be watched/unwatched & creates or deletes any queue records and updated Document record `watched` status
+   * @param {import("@prisma/client").workspace_documents} documentRecord
+   * @param {boolean} watchStatus - indicate if queue record should be created or not.
+   * @returns
+   */
+  toggleWatchStatus: async function (documentRecord, watchStatus = false) {
+    if (!watchStatus) {
+      await this.unwatch(documentRecord);
+      return;
+    }
+
+    await this.watch(documentRecord);
+    return;
+  },
+};
+
+module.exports = { DocumentSyncQueue };

package/server/models/documentSyncRun.js

@@ -0,0 +1,88 @@
+const prisma = require("../utils/prisma");
+const DocumentSyncRun = {
+  statuses: {
+    unknown: "unknown",
+    exited: "exited",
+    failed: "failed",
+    success: "success",
+  },
+
+  save: async function (queueId = null, status = null, result = {}) {
+    try {
+      if (!this.statuses.hasOwnProperty(status))
+        throw new Error(
+          `DocumentSyncRun status ${status} is not a valid status.`
+        );
+
+      const run = await prisma.document_sync_executions.create({
+        data: {
+          queueId: Number(queueId),
+          status: String(status),
+          result: JSON.stringify(result),
+        },
+      });
+      return run || null;
+    } catch (error) {
+      console.error(error.message);
+      return null;
+    }
+  },
+
+  get: async function (clause = {}) {
+    try {
+      const queue = await prisma.document_sync_executions.findFirst({
+        where: clause,
+      });
+      return queue || null;
+    } catch (error) {
+      console.error(error.message);
+      return null;
+    }
+  },
+
+  where: async function (
+    clause = {},
+    limit = null,
+    orderBy = null,
+    include = {}
+  ) {
+    try {
+      const results = await prisma.document_sync_executions.findMany({
+        where: clause,
+        ...(limit !== null ? { take: limit } : {}),
+        ...(orderBy !== null ? { orderBy } : {}),
+        ...(include !== null ? { include } : {}),
+      });
+      return results;
+    } catch (error) {
+      console.error(error.message);
+      return [];
+    }
+  },
+
+  count: async function (clause = {}, limit = null, orderBy = {}) {
+    try {
+      const count = await prisma.document_sync_executions.count({
+        where: clause,
+        ...(limit !== null ? { take: limit } : {}),
+        ...(orderBy !== null ? { orderBy } : {}),
+      });
+      return count;
+    } catch (error) {
+      console.error("FAILED TO COUNT DOCUMENTS.", error.message);
+      return 0;
+    }
+  },
+
+  delete: async function (clause = {}) {
+    try {
+      await prisma.document_sync_executions.deleteMany({ where: clause });
+      return true;
+    } catch (error) {
+      console.error(error.message);
+      return false;
+    }
+  },
+};
+
+module.exports = { DocumentSyncRun };