totem-llm 0.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (800) hide show
  1. package/LICENSE +22 -0
  2. package/README.md +81 -0
  3. package/bin/totem-llm.js +80 -0
  4. package/collector/.env +9 -0
  5. package/collector/.env.example +9 -0
  6. package/collector/.nvmrc +1 -0
  7. package/collector/extensions/index.js +239 -0
  8. package/collector/extensions/resync/index.js +197 -0
  9. package/collector/hotdir/__HOTDIR__.md +3 -0
  10. package/collector/index.js +202 -0
  11. package/collector/middleware/httpLogger.js +29 -0
  12. package/collector/middleware/setDataSigner.js +44 -0
  13. package/collector/middleware/verifyIntegrity.js +32 -0
  14. package/collector/package.json +67 -0
  15. package/collector/processLink/convert/generic.js +240 -0
  16. package/collector/processLink/helpers/index.js +190 -0
  17. package/collector/processLink/index.js +47 -0
  18. package/collector/processRawText/index.js +83 -0
  19. package/collector/processSingleFile/convert/asAudio.js +79 -0
  20. package/collector/processSingleFile/convert/asDocx.js +63 -0
  21. package/collector/processSingleFile/convert/asEPub.js +61 -0
  22. package/collector/processSingleFile/convert/asImage.js +56 -0
  23. package/collector/processSingleFile/convert/asMbox.js +83 -0
  24. package/collector/processSingleFile/convert/asOfficeMime.js +59 -0
  25. package/collector/processSingleFile/convert/asPDF/PDFLoader/index.js +97 -0
  26. package/collector/processSingleFile/convert/asPDF/index.js +86 -0
  27. package/collector/processSingleFile/convert/asTxt.js +59 -0
  28. package/collector/processSingleFile/convert/asXlsx.js +193 -0
  29. package/collector/processSingleFile/index.js +95 -0
  30. package/collector/utils/EncryptionWorker/index.js +77 -0
  31. package/collector/utils/OCRLoader/index.js +355 -0
  32. package/collector/utils/OCRLoader/validLangs.js +155 -0
  33. package/collector/utils/WhisperProviders/OpenAiWhisper.js +49 -0
  34. package/collector/utils/WhisperProviders/ffmpeg/index.js +114 -0
  35. package/collector/utils/WhisperProviders/localWhisper.js +197 -0
  36. package/collector/utils/comKey/index.js +54 -0
  37. package/collector/utils/constants.js +92 -0
  38. package/collector/utils/downloadURIToFile/index.js +90 -0
  39. package/collector/utils/extensions/Confluence/ConfluenceLoader/index.js +154 -0
  40. package/collector/utils/extensions/Confluence/index.js +292 -0
  41. package/collector/utils/extensions/DrupalWiki/DrupalWiki/index.js +338 -0
  42. package/collector/utils/extensions/DrupalWiki/index.js +102 -0
  43. package/collector/utils/extensions/ObsidianVault/index.js +95 -0
  44. package/collector/utils/extensions/PaperlessNgx/PaperlessNgxLoader/index.js +157 -0
  45. package/collector/utils/extensions/PaperlessNgx/index.js +128 -0
  46. package/collector/utils/extensions/RepoLoader/GithubRepo/RepoLoader/index.js +265 -0
  47. package/collector/utils/extensions/RepoLoader/GithubRepo/index.js +159 -0
  48. package/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js +447 -0
  49. package/collector/utils/extensions/RepoLoader/GitlabRepo/index.js +261 -0
  50. package/collector/utils/extensions/RepoLoader/index.js +41 -0
  51. package/collector/utils/extensions/WebsiteDepth/index.js +211 -0
  52. package/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/index.js +99 -0
  53. package/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js +246 -0
  54. package/collector/utils/extensions/YoutubeTranscript/index.js +219 -0
  55. package/collector/utils/files/index.js +240 -0
  56. package/collector/utils/files/mime.js +64 -0
  57. package/collector/utils/http/index.js +54 -0
  58. package/collector/utils/logger/index.js +68 -0
  59. package/collector/utils/runtimeSettings/index.js +97 -0
  60. package/collector/utils/shell.js +25 -0
  61. package/collector/utils/tokenizer/index.js +66 -0
  62. package/collector/utils/url/index.js +137 -0
  63. package/collector/yarn.lock +4413 -0
  64. package/lib/config.js +79 -0
  65. package/lib/launcher.js +116 -0
  66. package/lib/setup.js +143 -0
  67. package/package.json +64 -0
  68. package/scripts/postinstall.mjs +117 -0
  69. package/server/.env +489 -0
  70. package/server/.env.example +496 -0
  71. package/server/.flowconfig +30 -0
  72. package/server/.nvmrc +1 -0
  73. package/server/endpoints/admin.js +566 -0
  74. package/server/endpoints/agentFileServer.js +174 -0
  75. package/server/endpoints/agentFlows.js +192 -0
  76. package/server/endpoints/agentSkillWhitelist.js +82 -0
  77. package/server/endpoints/agentWebsocket.js +62 -0
  78. package/server/endpoints/api/admin/index.js +775 -0
  79. package/server/endpoints/api/auth/index.js +33 -0
  80. package/server/endpoints/api/document/index.js +1210 -0
  81. package/server/endpoints/api/embed/index.js +409 -0
  82. package/server/endpoints/api/index.js +29 -0
  83. package/server/endpoints/api/openai/compatibility-test-script.cjs +79 -0
  84. package/server/endpoints/api/openai/helpers.js +50 -0
  85. package/server/endpoints/api/openai/index.js +321 -0
  86. package/server/endpoints/api/system/index.js +275 -0
  87. package/server/endpoints/api/userManagement/index.js +124 -0
  88. package/server/endpoints/api/workspace/index.js +1008 -0
  89. package/server/endpoints/api/workspaceThread/index.js +627 -0
  90. package/server/endpoints/browserExtension.js +221 -0
  91. package/server/endpoints/chat.js +193 -0
  92. package/server/endpoints/communityHub.js +214 -0
  93. package/server/endpoints/document.js +111 -0
  94. package/server/endpoints/embed/index.js +103 -0
  95. package/server/endpoints/embedManagement.js +131 -0
  96. package/server/endpoints/experimental/imported-agent-plugins.js +65 -0
  97. package/server/endpoints/experimental/index.js +12 -0
  98. package/server/endpoints/experimental/liveSync.js +110 -0
  99. package/server/endpoints/extensions/index.js +175 -0
  100. package/server/endpoints/invite.js +81 -0
  101. package/server/endpoints/mcpServers.js +127 -0
  102. package/server/endpoints/memory.js +196 -0
  103. package/server/endpoints/mobile/index.js +160 -0
  104. package/server/endpoints/mobile/middleware/index.js +97 -0
  105. package/server/endpoints/mobile/utils/index.js +195 -0
  106. package/server/endpoints/modelRouter.js +205 -0
  107. package/server/endpoints/scheduledJobs.js +374 -0
  108. package/server/endpoints/system.js +1490 -0
  109. package/server/endpoints/telegram.js +333 -0
  110. package/server/endpoints/utils/dockerModelRunnerUtils.js +94 -0
  111. package/server/endpoints/utils/googleAgentSkillEndpoints.js +70 -0
  112. package/server/endpoints/utils/lemonadeUtilsEndpoints.js +170 -0
  113. package/server/endpoints/utils/outlookAgentUtils.js +190 -0
  114. package/server/endpoints/utils.js +220 -0
  115. package/server/endpoints/webPush.js +27 -0
  116. package/server/endpoints/workspaceThreads.js +245 -0
  117. package/server/endpoints/workspaces.js +1159 -0
  118. package/server/endpoints/workspacesParsedFiles.js +208 -0
  119. package/server/index.js +178 -0
  120. package/server/jobs/cleanup-generated-files.js +160 -0
  121. package/server/jobs/cleanup-orphan-documents.js +79 -0
  122. package/server/jobs/embedding-worker.js +197 -0
  123. package/server/jobs/extract-memories.js +192 -0
  124. package/server/jobs/handle-telegram-chat.js +67 -0
  125. package/server/jobs/helpers/index.js +54 -0
  126. package/server/jobs/helpers/memory-extraction-utils.js +366 -0
  127. package/server/jobs/helpers/scheduled-job-helper.js +111 -0
  128. package/server/jobs/run-scheduled-job.js +157 -0
  129. package/server/jobs/sync-watched-documents.js +209 -0
  130. package/server/jsconfig.json +14 -0
  131. package/server/middleware/httpLogger.js +23 -0
  132. package/server/models/agentSkillWhitelist.js +100 -0
  133. package/server/models/apiKeys.js +99 -0
  134. package/server/models/browserExtensionApiKey.js +190 -0
  135. package/server/models/cacheData.js +69 -0
  136. package/server/models/communityHub.js +213 -0
  137. package/server/models/documentSyncQueue.js +252 -0
  138. package/server/models/documentSyncRun.js +88 -0
  139. package/server/models/documents.js +356 -0
  140. package/server/models/embedChats.js +199 -0
  141. package/server/models/embedConfig.js +251 -0
  142. package/server/models/eventLogs.js +129 -0
  143. package/server/models/externalCommunicationConnector.js +110 -0
  144. package/server/models/invite.js +144 -0
  145. package/server/models/memory.js +451 -0
  146. package/server/models/mobileDevice.js +251 -0
  147. package/server/models/modelRouter.js +259 -0
  148. package/server/models/modelRouterRule.js +336 -0
  149. package/server/models/passwordRecovery.js +115 -0
  150. package/server/models/promptHistory.js +107 -0
  151. package/server/models/scheduledJob.js +499 -0
  152. package/server/models/scheduledJobRun.js +362 -0
  153. package/server/models/slashCommandsPresets.js +143 -0
  154. package/server/models/systemPromptVariables.js +361 -0
  155. package/server/models/systemSettings.js +1183 -0
  156. package/server/models/temporaryAuthToken.js +104 -0
  157. package/server/models/user.js +381 -0
  158. package/server/models/vectors.js +79 -0
  159. package/server/models/workspace.js +720 -0
  160. package/server/models/workspaceAgentInvocation.js +97 -0
  161. package/server/models/workspaceChats.js +386 -0
  162. package/server/models/workspaceParsedFiles.js +246 -0
  163. package/server/models/workspaceThread.js +151 -0
  164. package/server/models/workspaceUsers.js +103 -0
  165. package/server/models/workspacesSuggestedMessages.js +83 -0
  166. package/server/package.json +130 -0
  167. package/server/prisma/migrations/20230921191814_init/migration.sql +125 -0
  168. package/server/prisma/migrations/20231101001441_init/migration.sql +2 -0
  169. package/server/prisma/migrations/20231101195421_init/migration.sql +11 -0
  170. package/server/prisma/migrations/20231129012019_add/migration.sql +2 -0
  171. package/server/prisma/migrations/20240113013409_init/migration.sql +2 -0
  172. package/server/prisma/migrations/20240118201333_init/migration.sql +2 -0
  173. package/server/prisma/migrations/20240202002020_init/migration.sql +37 -0
  174. package/server/prisma/migrations/20240206181106_init/migration.sql +13 -0
  175. package/server/prisma/migrations/20240206211916_init/migration.sql +11 -0
  176. package/server/prisma/migrations/20240208224848_init/migration.sql +24 -0
  177. package/server/prisma/migrations/20240210004405_init/migration.sql +2 -0
  178. package/server/prisma/migrations/20240216214639_init/migration.sql +2 -0
  179. package/server/prisma/migrations/20240219211018_init/migration.sql +2 -0
  180. package/server/prisma/migrations/20240301002308_init/migration.sql +2 -0
  181. package/server/prisma/migrations/20240326231053_init/migration.sql +2 -0
  182. package/server/prisma/migrations/20240405015034_init/migration.sql +2 -0
  183. package/server/prisma/migrations/20240412183346_init/migration.sql +24 -0
  184. package/server/prisma/migrations/20240425004220_init/migration.sql +30 -0
  185. package/server/prisma/migrations/20240430230707_init/migration.sql +2 -0
  186. package/server/prisma/migrations/20240510032311_init/migration.sql +15 -0
  187. package/server/prisma/migrations/20240618224346_init/migration.sql +26 -0
  188. package/server/prisma/migrations/20240821215625_init/migration.sql +2 -0
  189. package/server/prisma/migrations/20240824005054_init/migration.sql +15 -0
  190. package/server/prisma/migrations/20241003192954_init/migration.sql +2 -0
  191. package/server/prisma/migrations/20241029203722_init/migration.sql +12 -0
  192. package/server/prisma/migrations/20241029233509_init/migration.sql +5 -0
  193. package/server/prisma/migrations/20250102204948_init/migration.sql +2 -0
  194. package/server/prisma/migrations/20250226005538_init/migration.sql +2 -0
  195. package/server/prisma/migrations/20250318154720_init/migration.sql +18 -0
  196. package/server/prisma/migrations/20250506214129_init/migration.sql +13 -0
  197. package/server/prisma/migrations/20250709230835_init/migration.sql +2 -0
  198. package/server/prisma/migrations/20250725194841_init/migration.sql +17 -0
  199. package/server/prisma/migrations/20250808171557_init/migration.sql +23 -0
  200. package/server/prisma/migrations/20260130040204_init/migration.sql +2 -0
  201. package/server/prisma/migrations/20260313192859_init/migration.sql +10 -0
  202. package/server/prisma/migrations/20260319202916_init/migration.sql +12 -0
  203. package/server/prisma/migrations/20260406120000_init/migration.sql +2 -0
  204. package/server/prisma/migrations/20260423191158_init/migration.sql +29 -0
  205. package/server/prisma/migrations/20260424013043_init/migration.sql +22 -0
  206. package/server/prisma/migrations/20260520160208_init/migration.sql +46 -0
  207. package/server/prisma/migrations/migration_lock.toml +3 -0
  208. package/server/prisma/schema.prisma +484 -0
  209. package/server/prisma/seed.js +31 -0
  210. package/server/public/KaTeX_AMS-Regular.eot +0 -0
  211. package/server/public/KaTeX_AMS-Regular.ttf +0 -0
  212. package/server/public/KaTeX_AMS-Regular.woff +0 -0
  213. package/server/public/KaTeX_AMS-Regular.woff2 +0 -0
  214. package/server/public/KaTeX_Caligraphic-Bold.eot +0 -0
  215. package/server/public/KaTeX_Caligraphic-Bold.ttf +0 -0
  216. package/server/public/KaTeX_Caligraphic-Bold.woff +0 -0
  217. package/server/public/KaTeX_Caligraphic-Bold.woff2 +0 -0
  218. package/server/public/KaTeX_Caligraphic-Regular.eot +0 -0
  219. package/server/public/KaTeX_Caligraphic-Regular.ttf +0 -0
  220. package/server/public/KaTeX_Caligraphic-Regular.woff +0 -0
  221. package/server/public/KaTeX_Caligraphic-Regular.woff2 +0 -0
  222. package/server/public/KaTeX_Fraktur-Bold.eot +0 -0
  223. package/server/public/KaTeX_Fraktur-Bold.ttf +0 -0
  224. package/server/public/KaTeX_Fraktur-Bold.woff +0 -0
  225. package/server/public/KaTeX_Fraktur-Bold.woff2 +0 -0
  226. package/server/public/KaTeX_Fraktur-Regular.eot +0 -0
  227. package/server/public/KaTeX_Fraktur-Regular.ttf +0 -0
  228. package/server/public/KaTeX_Fraktur-Regular.woff +0 -0
  229. package/server/public/KaTeX_Fraktur-Regular.woff2 +0 -0
  230. package/server/public/KaTeX_Main-Bold.eot +0 -0
  231. package/server/public/KaTeX_Main-Bold.ttf +0 -0
  232. package/server/public/KaTeX_Main-Bold.woff +0 -0
  233. package/server/public/KaTeX_Main-Bold.woff2 +0 -0
  234. package/server/public/KaTeX_Main-Italic.eot +0 -0
  235. package/server/public/KaTeX_Main-Italic.ttf +0 -0
  236. package/server/public/KaTeX_Main-Italic.woff +0 -0
  237. package/server/public/KaTeX_Main-Italic.woff2 +0 -0
  238. package/server/public/KaTeX_Main-Regular.eot +0 -0
  239. package/server/public/KaTeX_Main-Regular.ttf +0 -0
  240. package/server/public/KaTeX_Main-Regular.woff +0 -0
  241. package/server/public/KaTeX_Main-Regular.woff2 +0 -0
  242. package/server/public/KaTeX_Math-Italic.eot +0 -0
  243. package/server/public/KaTeX_Math-Italic.ttf +0 -0
  244. package/server/public/KaTeX_Math-Italic.woff +0 -0
  245. package/server/public/KaTeX_Math-Italic.woff2 +0 -0
  246. package/server/public/KaTeX_SansSerif-Regular.eot +0 -0
  247. package/server/public/KaTeX_SansSerif-Regular.ttf +0 -0
  248. package/server/public/KaTeX_SansSerif-Regular.woff +0 -0
  249. package/server/public/KaTeX_SansSerif-Regular.woff2 +0 -0
  250. package/server/public/KaTeX_Script-Regular.eot +0 -0
  251. package/server/public/KaTeX_Script-Regular.ttf +0 -0
  252. package/server/public/KaTeX_Script-Regular.woff +0 -0
  253. package/server/public/KaTeX_Script-Regular.woff2 +0 -0
  254. package/server/public/KaTeX_Size1-Regular.eot +0 -0
  255. package/server/public/KaTeX_Size1-Regular.ttf +0 -0
  256. package/server/public/KaTeX_Size1-Regular.woff +0 -0
  257. package/server/public/KaTeX_Size1-Regular.woff2 +0 -0
  258. package/server/public/KaTeX_Size2-Regular.eot +0 -0
  259. package/server/public/KaTeX_Size2-Regular.ttf +0 -0
  260. package/server/public/KaTeX_Size2-Regular.woff +0 -0
  261. package/server/public/KaTeX_Size2-Regular.woff2 +0 -0
  262. package/server/public/KaTeX_Size3-Regular.eot +0 -0
  263. package/server/public/KaTeX_Size3-Regular.ttf +0 -0
  264. package/server/public/KaTeX_Size3-Regular.woff +0 -0
  265. package/server/public/KaTeX_Size4-Regular.eot +0 -0
  266. package/server/public/KaTeX_Size4-Regular.ttf +0 -0
  267. package/server/public/KaTeX_Size4-Regular.woff +0 -0
  268. package/server/public/KaTeX_Size4-Regular.woff2 +0 -0
  269. package/server/public/KaTeX_Typewriter-Regular.eot +0 -0
  270. package/server/public/KaTeX_Typewriter-Regular.ttf +0 -0
  271. package/server/public/KaTeX_Typewriter-Regular.woff +0 -0
  272. package/server/public/KaTeX_Typewriter-Regular.woff2 +0 -0
  273. package/server/public/PlusJakartaSans.ttf +0 -0
  274. package/server/public/_index.html +45 -0
  275. package/server/public/agent-animation.webm +0 -0
  276. package/server/public/agent-static.png +0 -0
  277. package/server/public/anthropic.png +0 -0
  278. package/server/public/anything-llm-dark.png +0 -0
  279. package/server/public/anything-llm-light.png +0 -0
  280. package/server/public/anything-llm.png +0 -0
  281. package/server/public/apipie.png +0 -0
  282. package/server/public/assets/404-aca8e9d6.js +1 -0
  283. package/server/public/assets/Brain-24e386d5.js +1 -0
  284. package/server/public/assets/Browser-1820fad0.js +1 -0
  285. package/server/public/assets/ChatText-35275002.js +1 -0
  286. package/server/public/assets/Check-7e83fb0d.js +1 -0
  287. package/server/public/assets/Copy-d323e545.js +1 -0
  288. package/server/public/assets/Database-f1a76329.js +1 -0
  289. package/server/public/assets/Eye-9f033dc0.js +1 -0
  290. package/server/public/assets/File-b92df2f3.js +1 -0
  291. package/server/public/assets/Gauge-e6762f4d.js +1 -0
  292. package/server/public/assets/Hammer-05b5ccc7.js +1 -0
  293. package/server/public/assets/Plug-7d951c7d.js +1 -0
  294. package/server/public/assets/PlusCircle-31c6a8d8.js +1 -0
  295. package/server/public/assets/RunDetailPage-edfdbd4d.js +1 -0
  296. package/server/public/assets/RunHistoryPage-732dae26.js +1 -0
  297. package/server/public/assets/Stop-4b47ae05.js +1 -0
  298. package/server/public/assets/Trans-36b04c7b.js +1 -0
  299. package/server/public/assets/WarningCircle-27467274.js +1 -0
  300. package/server/public/assets/XCircle-edfe554c.js +1 -0
  301. package/server/public/assets/admin-04b020fe.js +1 -0
  302. package/server/public/assets/agentFlows-e4124dd3.js +1 -0
  303. package/server/public/assets/anything-llm-infinity-0d7244a1.js +1 -0
  304. package/server/public/assets/communityHub-aba61686.js +1 -0
  305. package/server/public/assets/cron-2ad9194f.js +1 -0
  306. package/server/public/assets/files-493643b0.js +23 -0
  307. package/server/public/assets/github-dark-57c241de.js +1 -0
  308. package/server/public/assets/index-08cd0bc2.js +9 -0
  309. package/server/public/assets/index-0bf8ea25.js +1 -0
  310. package/server/public/assets/index-25b2660c.js +1 -0
  311. package/server/public/assets/index-2a7c0b04.js +1 -0
  312. package/server/public/assets/index-3643e990.js +1 -0
  313. package/server/public/assets/index-3761d55d.js +1 -0
  314. package/server/public/assets/index-4b5e1c76.js +5 -0
  315. package/server/public/assets/index-4fb4d421.js +2 -0
  316. package/server/public/assets/index-56bb11a0.js +1 -0
  317. package/server/public/assets/index-5c320a5d.js +1 -0
  318. package/server/public/assets/index-62160652.js +3 -0
  319. package/server/public/assets/index-62fee0f9.js +4 -0
  320. package/server/public/assets/index-65801ac6.js +1 -0
  321. package/server/public/assets/index-71027183.js +1 -0
  322. package/server/public/assets/index-72832304.js +18 -0
  323. package/server/public/assets/index-7b0ea06b.js +14 -0
  324. package/server/public/assets/index-8132c765.js +1 -0
  325. package/server/public/assets/index-837868b3.js +1 -0
  326. package/server/public/assets/index-86e8a6b4.js +1 -0
  327. package/server/public/assets/index-8ba96f42.js +1 -0
  328. package/server/public/assets/index-8e43159f.js +1 -0
  329. package/server/public/assets/index-92574f3a.js +1 -0
  330. package/server/public/assets/index-925792fe.js +2 -0
  331. package/server/public/assets/index-92d3b7bc.js +1 -0
  332. package/server/public/assets/index-994c849a.js +1 -0
  333. package/server/public/assets/index-9f71774b.js +1 -0
  334. package/server/public/assets/index-9f7dabd6.js +1 -0
  335. package/server/public/assets/index-a0e2be61.js +1 -0
  336. package/server/public/assets/index-a6ac5511.js +1 -0
  337. package/server/public/assets/index-a6c0d98a.js +1 -0
  338. package/server/public/assets/index-aa687cfe.js +3 -0
  339. package/server/public/assets/index-ab35fb74.js +1 -0
  340. package/server/public/assets/index-ab72b353.js +4 -0
  341. package/server/public/assets/index-ad8e77d2.js +5 -0
  342. package/server/public/assets/index-c07cc087.js +110 -0
  343. package/server/public/assets/index-c22f7da1.js +1 -0
  344. package/server/public/assets/index-c9b59c35.js +1 -0
  345. package/server/public/assets/index-d59c6d7f.js +1 -0
  346. package/server/public/assets/index-d69f2e09.js +1 -0
  347. package/server/public/assets/index-e5b911ea.js +1 -0
  348. package/server/public/assets/index-e6c540e1.js +1 -0
  349. package/server/public/assets/index-f86e167c.js +4 -0
  350. package/server/public/assets/index-f984ef26.js +1 -0
  351. package/server/public/assets/index-fb26b60a.js +1 -0
  352. package/server/public/assets/index-fd922432.js +1 -0
  353. package/server/public/assets/main-8ad4162f.js +1 -0
  354. package/server/public/assets/mcpServers-957b1f1c.js +1 -0
  355. package/server/public/assets/moment-3ad2e1a6.js +10 -0
  356. package/server/public/assets/ort.min-faca3c2b.js +1827 -0
  357. package/server/public/assets/piper-o91UDS6e-0c78e94e.js +4 -0
  358. package/server/public/assets/purify-6b86d6dc.js +26 -0
  359. package/server/public/assets/react-beautiful-dnd.esm-86e87aba.js +68 -0
  360. package/server/public/assets/truncate-352eb33b.js +1 -0
  361. package/server/public/assets/types-e90e1137.js +1 -0
  362. package/server/public/assets/usePolling-417bb099.js +1 -0
  363. package/server/public/assets/voices_static-D_OtJDHM-5cdb0950.js +1 -0
  364. package/server/public/assets/worker-8f433525.js +1 -0
  365. package/server/public/azure.png +0 -0
  366. package/server/public/baidu.png +0 -0
  367. package/server/public/bedrock.png +0 -0
  368. package/server/public/bg.png +0 -0
  369. package/server/public/bing.png +0 -0
  370. package/server/public/cerebras.png +0 -0
  371. package/server/public/cohere.png +0 -0
  372. package/server/public/cometapi.png +0 -0
  373. package/server/public/community-hub.png +0 -0
  374. package/server/public/confluence.jpeg +0 -0
  375. package/server/public/deepseek.png +0 -0
  376. package/server/public/docker-model-runner.png +0 -0
  377. package/server/public/dpais.png +0 -0
  378. package/server/public/drupalwiki.jpg +0 -0
  379. package/server/public/duckduckgo.png +0 -0
  380. package/server/public/elevenlabs.png +0 -0
  381. package/server/public/embed/anythingllm-chat-widget.min.css +1 -0
  382. package/server/public/embed/anythingllm-chat-widget.min.js +37 -0
  383. package/server/public/exa.png +0 -0
  384. package/server/public/favicon.ico +0 -0
  385. package/server/public/favicon.png +0 -0
  386. package/server/public/file-system.png +0 -0
  387. package/server/public/files.css +1 -0
  388. package/server/public/fonts/PlusJakartaSans.ttf +0 -0
  389. package/server/public/foundry-local.png +0 -0
  390. package/server/public/gemini.png +0 -0
  391. package/server/public/generate-charts.png +0 -0
  392. package/server/public/generate-save-files.png +0 -0
  393. package/server/public/generic-openai.png +0 -0
  394. package/server/public/giteeai.png +0 -0
  395. package/server/public/github-dark.css +10 -0
  396. package/server/public/github.css +19 -0
  397. package/server/public/github.svg +4 -0
  398. package/server/public/gitlab.svg +7 -0
  399. package/server/public/gmail.png +0 -0
  400. package/server/public/google-calendar.png +0 -0
  401. package/server/public/gplay-badge.svg +55 -0
  402. package/server/public/huggingface.png +0 -0
  403. package/server/public/index.css +1 -0
  404. package/server/public/index.js +1739 -0
  405. package/server/public/koboldcpp.png +0 -0
  406. package/server/public/lancedb.png +0 -0
  407. package/server/public/lemonade.png +0 -0
  408. package/server/public/link.svg +1 -0
  409. package/server/public/litellm.png +0 -0
  410. package/server/public/lmstudio.png +0 -0
  411. package/server/public/localai.png +0 -0
  412. package/server/public/login-logo-light.svg +45 -0
  413. package/server/public/login-logo.svg +45 -0
  414. package/server/public/manifest.json +13 -0
  415. package/server/public/mcp-logo.svg +1 -0
  416. package/server/public/milvus.png +0 -0
  417. package/server/public/minimax.png +0 -0
  418. package/server/public/mistral.jpeg +0 -0
  419. package/server/public/moonshotai.png +0 -0
  420. package/server/public/mssql.png +0 -0
  421. package/server/public/mysql.png +0 -0
  422. package/server/public/nvidia-nim.png +0 -0
  423. package/server/public/obsidian.png +0 -0
  424. package/server/public/ollama.png +0 -0
  425. package/server/public/openai.png +0 -0
  426. package/server/public/openrouter.jpeg +0 -0
  427. package/server/public/outlook.png +0 -0
  428. package/server/public/paperless-ngx.jpeg +0 -0
  429. package/server/public/perplexity.png +0 -0
  430. package/server/public/pgvector.png +0 -0
  431. package/server/public/piper.png +0 -0
  432. package/server/public/privatemode.png +0 -0
  433. package/server/public/qdrant.png +0 -0
  434. package/server/public/rag-memory.png +0 -0
  435. package/server/public/robots.txt +2 -0
  436. package/server/public/router-animation.webm +0 -0
  437. package/server/public/sambanova.png +0 -0
  438. package/server/public/scrape-websites.png +0 -0
  439. package/server/public/searchapi.png +0 -0
  440. package/server/public/searxng.png +0 -0
  441. package/server/public/serpapi.png +0 -0
  442. package/server/public/serper.png +0 -0
  443. package/server/public/serply.png +0 -0
  444. package/server/public/service-workers/push-notifications.js +26 -0
  445. package/server/public/sql-agent.png +0 -0
  446. package/server/public/tavily.svg +1 -0
  447. package/server/public/text-generation-webui.png +0 -0
  448. package/server/public/thinking-animation.webm +0 -0
  449. package/server/public/thinking-static.png +0 -0
  450. package/server/public/togetherai.png +0 -0
  451. package/server/public/view-summarize.png +0 -0
  452. package/server/public/voyageai.png +0 -0
  453. package/server/public/weaviate.png +0 -0
  454. package/server/public/xai.png +0 -0
  455. package/server/public/youtube.svg +10 -0
  456. package/server/public/zai.png +0 -0
  457. package/server/public/zilliz.png +0 -0
  458. package/server/utils/AiProviders/anthropic/index.js +372 -0
  459. package/server/utils/AiProviders/apipie/index.js +386 -0
  460. package/server/utils/AiProviders/azureOpenAi/index.js +232 -0
  461. package/server/utils/AiProviders/bedrock/index.js +787 -0
  462. package/server/utils/AiProviders/bedrock/utils.js +166 -0
  463. package/server/utils/AiProviders/cerebras/index.js +314 -0
  464. package/server/utils/AiProviders/cohere/index.js +251 -0
  465. package/server/utils/AiProviders/cometapi/constants.js +39 -0
  466. package/server/utils/AiProviders/cometapi/index.js +445 -0
  467. package/server/utils/AiProviders/deepseek/index.js +180 -0
  468. package/server/utils/AiProviders/dellProAiStudio/index.js +217 -0
  469. package/server/utils/AiProviders/dockerModelRunner/index.js +520 -0
  470. package/server/utils/AiProviders/fireworksAi/index.js +273 -0
  471. package/server/utils/AiProviders/foundry/index.js +475 -0
  472. package/server/utils/AiProviders/gemini/defaultModels.js +74 -0
  473. package/server/utils/AiProviders/gemini/index.js +457 -0
  474. package/server/utils/AiProviders/gemini/syncStaticLists.mjs +48 -0
  475. package/server/utils/AiProviders/genericOpenAi/index.js +485 -0
  476. package/server/utils/AiProviders/giteeai/index.js +245 -0
  477. package/server/utils/AiProviders/groq/index.js +257 -0
  478. package/server/utils/AiProviders/huggingface/index.js +165 -0
  479. package/server/utils/AiProviders/koboldCPP/index.js +264 -0
  480. package/server/utils/AiProviders/lemonade/index.js +463 -0
  481. package/server/utils/AiProviders/liteLLM/index.js +201 -0
  482. package/server/utils/AiProviders/lmStudio/index.js +383 -0
  483. package/server/utils/AiProviders/localAi/index.js +198 -0
  484. package/server/utils/AiProviders/minimax/index.js +165 -0
  485. package/server/utils/AiProviders/mistral/index.js +191 -0
  486. package/server/utils/AiProviders/modelMap/index.js +212 -0
  487. package/server/utils/AiProviders/modelMap/legacy.js +162 -0
  488. package/server/utils/AiProviders/modelRouter/index.js +170 -0
  489. package/server/utils/AiProviders/moonshotAi/index.js +176 -0
  490. package/server/utils/AiProviders/novita/index.js +460 -0
  491. package/server/utils/AiProviders/nvidiaNim/index.js +253 -0
  492. package/server/utils/AiProviders/ollama/index.js +516 -0
  493. package/server/utils/AiProviders/openAi/index.js +301 -0
  494. package/server/utils/AiProviders/openRouter/index.js +567 -0
  495. package/server/utils/AiProviders/perplexity/index.js +304 -0
  496. package/server/utils/AiProviders/perplexity/models.js +24 -0
  497. package/server/utils/AiProviders/perplexity/scripts/chat_models.txt +6 -0
  498. package/server/utils/AiProviders/perplexity/scripts/parse.mjs +49 -0
  499. package/server/utils/AiProviders/ppio/index.js +274 -0
  500. package/server/utils/AiProviders/privatemode/index.js +220 -0
  501. package/server/utils/AiProviders/sambanova/index.js +273 -0
  502. package/server/utils/AiProviders/textGenWebUI/index.js +197 -0
  503. package/server/utils/AiProviders/togetherAi/index.js +263 -0
  504. package/server/utils/AiProviders/xai/index.js +200 -0
  505. package/server/utils/AiProviders/zai/index.js +184 -0
  506. package/server/utils/BackgroundWorkers/index.js +432 -0
  507. package/server/utils/DocumentManager/index.js +72 -0
  508. package/server/utils/EmbeddingEngines/azureOpenAi/index.js +115 -0
  509. package/server/utils/EmbeddingEngines/cohere/index.js +91 -0
  510. package/server/utils/EmbeddingEngines/gemini/index.js +136 -0
  511. package/server/utils/EmbeddingEngines/genericOpenAi/index.js +128 -0
  512. package/server/utils/EmbeddingEngines/lemonade/index.js +99 -0
  513. package/server/utils/EmbeddingEngines/liteLLM/index.js +102 -0
  514. package/server/utils/EmbeddingEngines/lmstudio/index.js +122 -0
  515. package/server/utils/EmbeddingEngines/localAi/index.js +122 -0
  516. package/server/utils/EmbeddingEngines/mistral/index.js +40 -0
  517. package/server/utils/EmbeddingEngines/native/constants.js +63 -0
  518. package/server/utils/EmbeddingEngines/native/index.js +290 -0
  519. package/server/utils/EmbeddingEngines/ollama/index.js +136 -0
  520. package/server/utils/EmbeddingEngines/openAi/index.js +102 -0
  521. package/server/utils/EmbeddingEngines/openRouter/index.js +131 -0
  522. package/server/utils/EmbeddingEngines/voyageAi/index.js +71 -0
  523. package/server/utils/EmbeddingRerankers/native/index.js +257 -0
  524. package/server/utils/EmbeddingWorkerManager.js +202 -0
  525. package/server/utils/EncryptionManager/index.js +85 -0
  526. package/server/utils/MCP/hypervisor/index.js +554 -0
  527. package/server/utils/MCP/index.js +269 -0
  528. package/server/utils/PasswordRecovery/index.js +103 -0
  529. package/server/utils/PushNotifications/index.js +228 -0
  530. package/server/utils/TextSplitter/index.js +206 -0
  531. package/server/utils/TextToSpeech/elevenLabs/index.js +54 -0
  532. package/server/utils/TextToSpeech/index.js +18 -0
  533. package/server/utils/TextToSpeech/openAi/index.js +29 -0
  534. package/server/utils/TextToSpeech/openAiGeneric/index.js +58 -0
  535. package/server/utils/agentFlows/executor.js +233 -0
  536. package/server/utils/agentFlows/executors/api-call.js +60 -0
  537. package/server/utils/agentFlows/executors/llm-instruction.js +47 -0
  538. package/server/utils/agentFlows/executors/web-scraping.js +110 -0
  539. package/server/utils/agentFlows/flowTypes.js +85 -0
  540. package/server/utils/agentFlows/index.js +288 -0
  541. package/server/utils/agents/aibitat/error.js +18 -0
  542. package/server/utils/agents/aibitat/example/beginner-chat.js +56 -0
  543. package/server/utils/agents/aibitat/example/blog-post-coding.js +55 -0
  544. package/server/utils/agents/aibitat/example/websocket/index.html +67 -0
  545. package/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js +100 -0
  546. package/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js +91 -0
  547. package/server/utils/agents/aibitat/index.js +1437 -0
  548. package/server/utils/agents/aibitat/plugins/chat-history.js +210 -0
  549. package/server/utils/agents/aibitat/plugins/cli.js +134 -0
  550. package/server/utils/agents/aibitat/plugins/create-files/assets/anything-llm-invert.png +0 -0
  551. package/server/utils/agents/aibitat/plugins/create-files/assets/anything-llm.png +0 -0
  552. package/server/utils/agents/aibitat/plugins/create-files/docx/create-docx-file.js +301 -0
  553. package/server/utils/agents/aibitat/plugins/create-files/docx/test-themes.js +298 -0
  554. package/server/utils/agents/aibitat/plugins/create-files/docx/utils.js +1095 -0
  555. package/server/utils/agents/aibitat/plugins/create-files/index.js +23 -0
  556. package/server/utils/agents/aibitat/plugins/create-files/lib.js +300 -0
  557. package/server/utils/agents/aibitat/plugins/create-files/pdf/create-pdf-file.js +138 -0
  558. package/server/utils/agents/aibitat/plugins/create-files/pdf/utils.js +70 -0
  559. package/server/utils/agents/aibitat/plugins/create-files/pptx/create-presentation.js +342 -0
  560. package/server/utils/agents/aibitat/plugins/create-files/pptx/section-agent.js +257 -0
  561. package/server/utils/agents/aibitat/plugins/create-files/pptx/test-themes.js +143 -0
  562. package/server/utils/agents/aibitat/plugins/create-files/pptx/themes.js +181 -0
  563. package/server/utils/agents/aibitat/plugins/create-files/pptx/utils.js +378 -0
  564. package/server/utils/agents/aibitat/plugins/create-files/text/create-text-file.js +158 -0
  565. package/server/utils/agents/aibitat/plugins/create-files/xlsx/create-excel-file.js +362 -0
  566. package/server/utils/agents/aibitat/plugins/create-files/xlsx/utils.js +333 -0
  567. package/server/utils/agents/aibitat/plugins/file-history.js +37 -0
  568. package/server/utils/agents/aibitat/plugins/filesystem/copy-file.js +127 -0
  569. package/server/utils/agents/aibitat/plugins/filesystem/create-directory.js +84 -0
  570. package/server/utils/agents/aibitat/plugins/filesystem/edit-file.js +133 -0
  571. package/server/utils/agents/aibitat/plugins/filesystem/get-file-info.js +75 -0
  572. package/server/utils/agents/aibitat/plugins/filesystem/index.js +33 -0
  573. package/server/utils/agents/aibitat/plugins/filesystem/lib.js +787 -0
  574. package/server/utils/agents/aibitat/plugins/filesystem/list-directory.js +175 -0
  575. package/server/utils/agents/aibitat/plugins/filesystem/move-file.js +95 -0
  576. package/server/utils/agents/aibitat/plugins/filesystem/read-multiple-files.js +157 -0
  577. package/server/utils/agents/aibitat/plugins/filesystem/read-text-file.js +142 -0
  578. package/server/utils/agents/aibitat/plugins/filesystem/search-files.js +469 -0
  579. package/server/utils/agents/aibitat/plugins/filesystem/write-text-file.js +90 -0
  580. package/server/utils/agents/aibitat/plugins/gmail/account/gmail-get-mailbox-stats.js +75 -0
  581. package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-create-draft-reply.js +220 -0
  582. package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-create-draft.js +217 -0
  583. package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-delete-draft.js +87 -0
  584. package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-get-draft.js +84 -0
  585. package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-list-drafts.js +96 -0
  586. package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-send-draft.js +94 -0
  587. package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-update-draft.js +217 -0
  588. package/server/utils/agents/aibitat/plugins/gmail/index.js +73 -0
  589. package/server/utils/agents/aibitat/plugins/gmail/lib.js +573 -0
  590. package/server/utils/agents/aibitat/plugins/gmail/search/gmail-get-inbox.js +104 -0
  591. package/server/utils/agents/aibitat/plugins/gmail/search/gmail-read-thread.js +122 -0
  592. package/server/utils/agents/aibitat/plugins/gmail/search/gmail-search.js +121 -0
  593. package/server/utils/agents/aibitat/plugins/gmail/send/gmail-reply-to-thread.js +238 -0
  594. package/server/utils/agents/aibitat/plugins/gmail/send/gmail-send-email.js +242 -0
  595. package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-mark-read.js +87 -0
  596. package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-mark-unread.js +87 -0
  597. package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-move-to-archive.js +89 -0
  598. package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-move-to-inbox.js +89 -0
  599. package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-move-to-trash.js +89 -0
  600. package/server/utils/agents/aibitat/plugins/google-calendar/calendars/gcal-get-calendar.js +79 -0
  601. package/server/utils/agents/aibitat/plugins/google-calendar/calendars/gcal-list-calendars.js +82 -0
  602. package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-create-event.js +233 -0
  603. package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-get-event.js +116 -0
  604. package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-get-events-for-day.js +117 -0
  605. package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-get-events.js +159 -0
  606. package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-get-upcoming-events.js +249 -0
  607. package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-quick-add.js +114 -0
  608. package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-set-my-status.js +125 -0
  609. package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-update-event.js +173 -0
  610. package/server/utils/agents/aibitat/plugins/google-calendar/index.js +40 -0
  611. package/server/utils/agents/aibitat/plugins/google-calendar/lib.js +288 -0
  612. package/server/utils/agents/aibitat/plugins/http-socket.js +249 -0
  613. package/server/utils/agents/aibitat/plugins/index.js +47 -0
  614. package/server/utils/agents/aibitat/plugins/memory.js +167 -0
  615. package/server/utils/agents/aibitat/plugins/outlook/account/outlook-get-mailbox-stats.js +78 -0
  616. package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-create-draft.js +232 -0
  617. package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-delete-draft.js +86 -0
  618. package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-list-drafts.js +80 -0
  619. package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-send-draft.js +82 -0
  620. package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-update-draft.js +132 -0
  621. package/server/utils/agents/aibitat/plugins/outlook/index.js +45 -0
  622. package/server/utils/agents/aibitat/plugins/outlook/lib.js +1413 -0
  623. package/server/utils/agents/aibitat/plugins/outlook/search/outlook-get-inbox.js +75 -0
  624. package/server/utils/agents/aibitat/plugins/outlook/search/outlook-read-thread.js +131 -0
  625. package/server/utils/agents/aibitat/plugins/outlook/search/outlook-search.js +101 -0
  626. package/server/utils/agents/aibitat/plugins/outlook/send/outlook-send-email.js +222 -0
  627. package/server/utils/agents/aibitat/plugins/rechart.js +114 -0
  628. package/server/utils/agents/aibitat/plugins/request-user-input.js +265 -0
  629. package/server/utils/agents/aibitat/plugins/router-classifier.js +122 -0
  630. package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MSSQL.js +114 -0
  631. package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MySQL.js +83 -0
  632. package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/Postgresql.js +75 -0
  633. package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/index.js +80 -0
  634. package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/utils.js +182 -0
  635. package/server/utils/agents/aibitat/plugins/sql-agent/get-table-schema.js +115 -0
  636. package/server/utils/agents/aibitat/plugins/sql-agent/index.js +21 -0
  637. package/server/utils/agents/aibitat/plugins/sql-agent/list-database.js +49 -0
  638. package/server/utils/agents/aibitat/plugins/sql-agent/list-table.js +105 -0
  639. package/server/utils/agents/aibitat/plugins/sql-agent/query.js +101 -0
  640. package/server/utils/agents/aibitat/plugins/summarize.js +189 -0
  641. package/server/utils/agents/aibitat/plugins/web-browsing.js +1182 -0
  642. package/server/utils/agents/aibitat/plugins/web-scraping.js +158 -0
  643. package/server/utils/agents/aibitat/plugins/websocket.js +361 -0
  644. package/server/utils/agents/aibitat/providers/ai-provider.js +706 -0
  645. package/server/utils/agents/aibitat/providers/anthropic.js +490 -0
  646. package/server/utils/agents/aibitat/providers/apipie.js +99 -0
  647. package/server/utils/agents/aibitat/providers/azure.js +122 -0
  648. package/server/utils/agents/aibitat/providers/bedrock.js +456 -0
  649. package/server/utils/agents/aibitat/providers/cerebras.js +208 -0
  650. package/server/utils/agents/aibitat/providers/cohere.js +269 -0
  651. package/server/utils/agents/aibitat/providers/cometapi.js +104 -0
  652. package/server/utils/agents/aibitat/providers/deepseek.js +223 -0
  653. package/server/utils/agents/aibitat/providers/dellProAiStudio.js +106 -0
  654. package/server/utils/agents/aibitat/providers/dockerModelRunner.js +189 -0
  655. package/server/utils/agents/aibitat/providers/fireworksai.js +101 -0
  656. package/server/utils/agents/aibitat/providers/foundry.js +112 -0
  657. package/server/utils/agents/aibitat/providers/gemini.js +450 -0
  658. package/server/utils/agents/aibitat/providers/genericOpenAi.js +206 -0
  659. package/server/utils/agents/aibitat/providers/giteeai.js +94 -0
  660. package/server/utils/agents/aibitat/providers/groq.js +184 -0
  661. package/server/utils/agents/aibitat/providers/helpers/classes.js +16 -0
  662. package/server/utils/agents/aibitat/providers/helpers/tooled.js +384 -0
  663. package/server/utils/agents/aibitat/providers/helpers/untooled.js +437 -0
  664. package/server/utils/agents/aibitat/providers/index.js +77 -0
  665. package/server/utils/agents/aibitat/providers/koboldcpp.js +103 -0
  666. package/server/utils/agents/aibitat/providers/lemonade.js +213 -0
  667. package/server/utils/agents/aibitat/providers/litellm.js +186 -0
  668. package/server/utils/agents/aibitat/providers/lmstudio.js +195 -0
  669. package/server/utils/agents/aibitat/providers/localai.js +190 -0
  670. package/server/utils/agents/aibitat/providers/minimax.js +194 -0
  671. package/server/utils/agents/aibitat/providers/mistral.js +103 -0
  672. package/server/utils/agents/aibitat/providers/moonshotAi.js +97 -0
  673. package/server/utils/agents/aibitat/providers/novita.js +188 -0
  674. package/server/utils/agents/aibitat/providers/nvidiaNim.js +101 -0
  675. package/server/utils/agents/aibitat/providers/ollama.js +606 -0
  676. package/server/utils/agents/aibitat/providers/openai.js +345 -0
  677. package/server/utils/agents/aibitat/providers/openrouter.js +195 -0
  678. package/server/utils/agents/aibitat/providers/perplexity.js +99 -0
  679. package/server/utils/agents/aibitat/providers/ppio.js +104 -0
  680. package/server/utils/agents/aibitat/providers/privatemode.js +107 -0
  681. package/server/utils/agents/aibitat/providers/sambanova.js +94 -0
  682. package/server/utils/agents/aibitat/providers/textgenwebui.js +99 -0
  683. package/server/utils/agents/aibitat/providers/togetherai.js +100 -0
  684. package/server/utils/agents/aibitat/providers/xai.js +99 -0
  685. package/server/utils/agents/aibitat/providers/zai.js +97 -0
  686. package/server/utils/agents/aibitat/utils/dedupe.js +152 -0
  687. package/server/utils/agents/aibitat/utils/summarize.js +63 -0
  688. package/server/utils/agents/aibitat/utils/toolReranker.js +225 -0
  689. package/server/utils/agents/defaults.js +196 -0
  690. package/server/utils/agents/ephemeral.js +790 -0
  691. package/server/utils/agents/imported-manifest.schema.json +158 -0
  692. package/server/utils/agents/imported.js +317 -0
  693. package/server/utils/agents/index.js +883 -0
  694. package/server/utils/boot/MetaGenerator.js +376 -0
  695. package/server/utils/boot/eagerLoadContextWindows.js +40 -0
  696. package/server/utils/boot/index.js +90 -0
  697. package/server/utils/boot/markOnboarded.js +52 -0
  698. package/server/utils/chats/agents.js +111 -0
  699. package/server/utils/chats/apiChatHandler.js +864 -0
  700. package/server/utils/chats/commands/reset.js +37 -0
  701. package/server/utils/chats/embed.js +301 -0
  702. package/server/utils/chats/index.js +128 -0
  703. package/server/utils/chats/openaiCompatible.js +537 -0
  704. package/server/utils/chats/stream.js +376 -0
  705. package/server/utils/collectorApi/index.js +319 -0
  706. package/server/utils/comKey/index.js +86 -0
  707. package/server/utils/database/index.js +83 -0
  708. package/server/utils/files/index.js +519 -0
  709. package/server/utils/files/logo.js +117 -0
  710. package/server/utils/files/multer.js +178 -0
  711. package/server/utils/files/pfp.js +63 -0
  712. package/server/utils/files/purgeDocument.js +91 -0
  713. package/server/utils/helpers/admin/index.js +56 -0
  714. package/server/utils/helpers/agents.js +35 -0
  715. package/server/utils/helpers/camelcase.js +143 -0
  716. package/server/utils/helpers/chat/LLMPerformanceMonitor.js +113 -0
  717. package/server/utils/helpers/chat/convertTo.js +256 -0
  718. package/server/utils/helpers/chat/index.js +448 -0
  719. package/server/utils/helpers/chat/responses.js +383 -0
  720. package/server/utils/helpers/customModels.js +1142 -0
  721. package/server/utils/helpers/index.js +716 -0
  722. package/server/utils/helpers/portAvailabilityChecker.js +46 -0
  723. package/server/utils/helpers/search.js +94 -0
  724. package/server/utils/helpers/shell.js +25 -0
  725. package/server/utils/helpers/tiktoken.js +107 -0
  726. package/server/utils/helpers/updateENV.js +1399 -0
  727. package/server/utils/http/index.js +141 -0
  728. package/server/utils/logger/index.js +66 -0
  729. package/server/utils/memories/index.js +139 -0
  730. package/server/utils/middleware/chatHistoryViewable.js +18 -0
  731. package/server/utils/middleware/communityHubDownloadsEnabled.js +77 -0
  732. package/server/utils/middleware/embedMiddleware.js +178 -0
  733. package/server/utils/middleware/featureFlagEnabled.js +24 -0
  734. package/server/utils/middleware/isSupportedRepoProviders.js +12 -0
  735. package/server/utils/middleware/multiUserProtected.js +107 -0
  736. package/server/utils/middleware/requireFeature.js +46 -0
  737. package/server/utils/middleware/simpleSSOEnabled.js +84 -0
  738. package/server/utils/middleware/validApiKey.js +29 -0
  739. package/server/utils/middleware/validBrowserExtensionApiKey.js +51 -0
  740. package/server/utils/middleware/validWorkspace.js +52 -0
  741. package/server/utils/middleware/validatedRequest.js +111 -0
  742. package/server/utils/middleware/workspaceDeletionProtection.js +18 -0
  743. package/server/utils/prisma/PRISMA.md +47 -0
  744. package/server/utils/prisma/index.js +13 -0
  745. package/server/utils/router/index.js +653 -0
  746. package/server/utils/telegramBot/chat/agent.js +425 -0
  747. package/server/utils/telegramBot/chat/stream.js +471 -0
  748. package/server/utils/telegramBot/constants.js +23 -0
  749. package/server/utils/telegramBot/index.js +844 -0
  750. package/server/utils/telegramBot/utils/commands/handlers/handleAbort.js +15 -0
  751. package/server/utils/telegramBot/utils/commands/handlers/handleHelp.js +15 -0
  752. package/server/utils/telegramBot/utils/commands/handlers/handleHistory.js +80 -0
  753. package/server/utils/telegramBot/utils/commands/handlers/handleNewThread.js +35 -0
  754. package/server/utils/telegramBot/utils/commands/handlers/handleProof.js +202 -0
  755. package/server/utils/telegramBot/utils/commands/handlers/handleReset.js +32 -0
  756. package/server/utils/telegramBot/utils/commands/handlers/handleStart.js +19 -0
  757. package/server/utils/telegramBot/utils/commands/handlers/handleStatus.js +80 -0
  758. package/server/utils/telegramBot/utils/commands/handlers/showModelMenu.js +98 -0
  759. package/server/utils/telegramBot/utils/commands/handlers/showThreadMenu.js +120 -0
  760. package/server/utils/telegramBot/utils/commands/handlers/showWorkspaceMenu.js +84 -0
  761. package/server/utils/telegramBot/utils/commands/index.js +125 -0
  762. package/server/utils/telegramBot/utils/format.js +225 -0
  763. package/server/utils/telegramBot/utils/index.js +203 -0
  764. package/server/utils/telegramBot/utils/media.js +163 -0
  765. package/server/utils/telegramBot/utils/messageQueue.js +53 -0
  766. package/server/utils/telegramBot/utils/navigation/callbacks/handleBackSources.js +16 -0
  767. package/server/utils/telegramBot/utils/navigation/callbacks/handleBackWorkspaces.js +18 -0
  768. package/server/utils/telegramBot/utils/navigation/callbacks/handleModelCancel.js +14 -0
  769. package/server/utils/telegramBot/utils/navigation/callbacks/handleModelPagination.js +24 -0
  770. package/server/utils/telegramBot/utils/navigation/callbacks/handleModelSelect.js +52 -0
  771. package/server/utils/telegramBot/utils/navigation/callbacks/handleSourcePagination.js +24 -0
  772. package/server/utils/telegramBot/utils/navigation/callbacks/handleSourceSelect.js +106 -0
  773. package/server/utils/telegramBot/utils/navigation/callbacks/handleThreadPagination.js +26 -0
  774. package/server/utils/telegramBot/utils/navigation/callbacks/handleThreadSelect.js +43 -0
  775. package/server/utils/telegramBot/utils/navigation/callbacks/handleToolApproval.js +104 -0
  776. package/server/utils/telegramBot/utils/navigation/callbacks/handleWorkspaceCreate.js +33 -0
  777. package/server/utils/telegramBot/utils/navigation/callbacks/handleWorkspacePagination.js +26 -0
  778. package/server/utils/telegramBot/utils/navigation/callbacks/handleWorkspaceSelect.js +24 -0
  779. package/server/utils/telegramBot/utils/navigation/callbacks/index.js +59 -0
  780. package/server/utils/telegramBot/utils/navigation/index.js +36 -0
  781. package/server/utils/telegramBot/utils/verification.js +171 -0
  782. package/server/utils/vectorDbProviders/astra/ASTRA_SETUP.md +22 -0
  783. package/server/utils/vectorDbProviders/astra/index.js +474 -0
  784. package/server/utils/vectorDbProviders/base.js +202 -0
  785. package/server/utils/vectorDbProviders/chroma/index.js +484 -0
  786. package/server/utils/vectorDbProviders/chromacloud/index.js +158 -0
  787. package/server/utils/vectorDbProviders/lance/index.js +506 -0
  788. package/server/utils/vectorDbProviders/milvus/MILVUS_SETUP.md +40 -0
  789. package/server/utils/vectorDbProviders/milvus/index.js +434 -0
  790. package/server/utils/vectorDbProviders/pgvector/SETUP.md +125 -0
  791. package/server/utils/vectorDbProviders/pgvector/index.js +850 -0
  792. package/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md +24 -0
  793. package/server/utils/vectorDbProviders/pinecone/index.js +317 -0
  794. package/server/utils/vectorDbProviders/qdrant/QDRANT_SETUP.md +17 -0
  795. package/server/utils/vectorDbProviders/qdrant/index.js +442 -0
  796. package/server/utils/vectorDbProviders/weaviate/WEAVIATE_SETUP.md +17 -0
  797. package/server/utils/vectorDbProviders/weaviate/index.js +510 -0
  798. package/server/utils/vectorDbProviders/zilliz/index.js +36 -0
  799. package/server/utils/vectorStore/resetAllVectorStores.js +59 -0
  800. package/server/yarn.lock +10702 -0
@@ -0,0 +1,86 @@
1
+ const crypto = require("crypto");
2
+ const fs = require("fs");
3
+ const path = require("path");
4
+ const keyPath =
5
+ process.env.NODE_ENV === "development"
6
+ ? path.resolve(__dirname, `../../storage/comkey`)
7
+ : path.resolve(
8
+ process.env.STORAGE_DIR ?? path.resolve(__dirname, `../../storage`),
9
+ `comkey`
10
+ );
11
+
12
+ // What does this class do?
13
+ // This class generates a hashed version of some text (typically a JSON payload) using a rolling RSA key
14
+ // that can then be appended as a header value to do integrity checking on a payload. Given the
15
+ // nature of this class and that keys are rolled constantly, this protects the request
16
+ // integrity of requests sent to the collector as only the server can sign these requests.
17
+ // This keeps accidental misconfigurations of AnythingLLM that leave the collector port open from
18
+ // being abused or SSRF'd by users scraping malicious sites who have a loopback embedded in a <script>, for example.
19
+ // Since each request to the collector must be signed to be valid, unsigned requests directly to the collector
20
+ // will be dropped and must go through the /server endpoint directly.
21
+ class CommunicationKey {
22
+ #privKeyName = "ipc-priv.pem";
23
+ #pubKeyName = "ipc-pub.pem";
24
+ #storageLoc = keyPath;
25
+
26
+ // Init the class and determine if keys should be rolled.
27
+ // This typically occurs on boot up so key is fresh each boot.
28
+ constructor(generate = false) {
29
+ if (generate) this.#generate();
30
+ }
31
+
32
+ log(text, ...args) {
33
+ console.log(`\x1b[36m[CommunicationKey]\x1b[0m ${text}`, ...args);
34
+ }
35
+
36
+ #readPrivateKey() {
37
+ return fs.readFileSync(path.resolve(this.#storageLoc, this.#privKeyName));
38
+ }
39
+
40
+ #generate() {
41
+ const keyPair = crypto.generateKeyPairSync("rsa", {
42
+ modulusLength: 2048,
43
+ publicKeyEncoding: {
44
+ type: "pkcs1",
45
+ format: "pem",
46
+ },
47
+ privateKeyEncoding: {
48
+ type: "pkcs1",
49
+ format: "pem",
50
+ },
51
+ });
52
+
53
+ if (!fs.existsSync(this.#storageLoc))
54
+ fs.mkdirSync(this.#storageLoc, { recursive: true });
55
+ fs.writeFileSync(
56
+ `${path.resolve(this.#storageLoc, this.#privKeyName)}`,
57
+ keyPair.privateKey
58
+ );
59
+ fs.writeFileSync(
60
+ `${path.resolve(this.#storageLoc, this.#pubKeyName)}`,
61
+ keyPair.publicKey
62
+ );
63
+ this.log(
64
+ "RSA key pair generated for signed payloads within AnythingLLM services."
65
+ );
66
+ }
67
+
68
+ // This instance of ComKey on server is intended for generation of Priv/Pub key for signing and decoding.
69
+ // this resource is shared with /collector/ via a class of the same name in /utils which does decoding/verification only
70
+ // while this server class only does signing with the private key.
71
+ sign(textData = "") {
72
+ return crypto
73
+ .sign("RSA-SHA256", Buffer.from(textData), this.#readPrivateKey())
74
+ .toString("hex");
75
+ }
76
+
77
+ // Use the rolling priv-key to encrypt arbitrary data that is text
78
+ // returns the encrypted content as a base64 string.
79
+ encrypt(textData = "") {
80
+ return crypto
81
+ .privateEncrypt(this.#readPrivateKey(), Buffer.from(textData, "utf-8"))
82
+ .toString("base64");
83
+ }
84
+ }
85
+
86
+ module.exports = { CommunicationKey };
@@ -0,0 +1,83 @@
1
+ const { getGitVersion } = require("../../endpoints/utils");
2
+
3
+ function checkColumnTemplate(tablename = null, column = null) {
4
+ if (!tablename || !column)
5
+ throw new Error(`Migration Error`, { tablename, column });
6
+ return `SELECT COUNT(*) AS _exists FROM pragma_table_info('${tablename}') WHERE name='${column}'`;
7
+ }
8
+
9
+ // Note (tcarambat): Since there is no good way to track migrations in Node/SQLite we use this simple system
10
+ // Each model has a `migrations` method that will return an array like...
11
+ // { colName: 'stringColName', execCmd: `SQL Command to run when`, doif: boolean },
12
+ // colName = name of column
13
+ // execCmd = Command to run when doif matches the state of the DB
14
+ // doif = condition to match that determines if execCmd will run.
15
+ // eg: Table workspace has slug column.
16
+ // execCmd: ALTER TABLE DROP COLUMN slug;
17
+ // doif: true
18
+ // => Will drop the slug column if the workspace table has a column named 'slug' otherwise nothing happens.
19
+ // If you are adding a new table column if needs to exist in the Models `colsInit` and as a migration.
20
+ // So both new and existing DBs will get the column when code is pulled in.
21
+
22
+ async function checkForMigrations(model, db) {
23
+ if (model.migrations().length === 0) return;
24
+ const toMigrate = [];
25
+ for (const { colName, execCmd, doif } of model.migrations()) {
26
+ const { _exists } = await db.get(
27
+ checkColumnTemplate(model.tablename, colName)
28
+ );
29
+ const colExists = _exists !== 0;
30
+ if (colExists !== doif) continue;
31
+
32
+ toMigrate.push(execCmd);
33
+ }
34
+
35
+ if (toMigrate.length === 0) return;
36
+
37
+ console.log(`Running ${toMigrate.length} migrations`, toMigrate);
38
+ await db.exec(toMigrate.join(";\n"));
39
+ return;
40
+ }
41
+
42
+ // Note(tcarambat): When building in production via Docker the SQLite file will not exist
43
+ // and if this function tries to run on boot the file will not exist
44
+ // and the server will abort and the container will exit.
45
+ // This function will run each reload on dev but on production
46
+ // it will be stubbed until the /api/migrate endpoint is GET.
47
+ async function validateTablePragmas(force = false) {
48
+ try {
49
+ if (process.env.NODE_ENV !== "development" && force === false) {
50
+ console.log(
51
+ `\x1b[34m[MIGRATIONS STUBBED]\x1b[0m Please ping /migrate once server starts to run migrations`
52
+ );
53
+ return;
54
+ }
55
+ const { SystemSettings } = require("../../models/systemSettings");
56
+ const { User } = require("../../models/user");
57
+ const { Workspace } = require("../../models/workspace");
58
+ const { WorkspaceUser } = require("../../models/workspaceUsers");
59
+ const { Document } = require("../../models/documents");
60
+ const { DocumentVectors } = require("../../models/vectors");
61
+ const { WorkspaceChats } = require("../../models/workspaceChats");
62
+ const { Invite } = require("../../models/invite");
63
+ const { ApiKey } = require("../../models/apiKeys");
64
+
65
+ await SystemSettings.migrateTable();
66
+ await User.migrateTable();
67
+ await Workspace.migrateTable();
68
+ await WorkspaceUser.migrateTable();
69
+ await Document.migrateTable();
70
+ await DocumentVectors.migrateTable();
71
+ await WorkspaceChats.migrateTable();
72
+ await Invite.migrateTable();
73
+ await ApiKey.migrateTable();
74
+ } catch (e) {
75
+ console.error(`validateTablePragmas: Migrations failed`, e);
76
+ }
77
+ return;
78
+ }
79
+
80
+ module.exports = {
81
+ checkForMigrations,
82
+ validateTablePragmas,
83
+ };
@@ -0,0 +1,519 @@
1
+ const fs = require("fs");
2
+ const path = require("path");
3
+ const { v5: uuidv5 } = require("uuid");
4
+ const { Document } = require("../../models/documents");
5
+ const { DocumentSyncQueue } = require("../../models/documentSyncQueue");
6
+ const documentsPath =
7
+ process.env.NODE_ENV === "development"
8
+ ? path.resolve(__dirname, `../../storage/documents`)
9
+ : path.resolve(process.env.STORAGE_DIR, `documents`);
10
+ const directUploadsPath =
11
+ process.env.NODE_ENV === "development"
12
+ ? path.resolve(__dirname, `../../storage/direct-uploads`)
13
+ : path.resolve(process.env.STORAGE_DIR, `direct-uploads`);
14
+ const vectorCachePath =
15
+ process.env.NODE_ENV === "development"
16
+ ? path.resolve(__dirname, `../../storage/vector-cache`)
17
+ : path.resolve(process.env.STORAGE_DIR, `vector-cache`);
18
+ const hotdirPath =
19
+ process.env.NODE_ENV === "development"
20
+ ? path.resolve(__dirname, `../../../collector/hotdir`)
21
+ : path.resolve(process.env.STORAGE_DIR, `../../collector/hotdir`);
22
+
23
+ // Should take in a folder that is a subfolder of documents
24
+ // eg: youtube-subject/video-123.json
25
+ async function fileData(filePath = null) {
26
+ if (!filePath) throw new Error("No docPath provided in request");
27
+ const fullFilePath = path.resolve(documentsPath, normalizePath(filePath));
28
+ if (!fs.existsSync(fullFilePath) || !isWithin(documentsPath, fullFilePath))
29
+ return null;
30
+
31
+ const data = fs.readFileSync(fullFilePath, "utf8");
32
+ return JSON.parse(data);
33
+ }
34
+
35
+ async function viewLocalFiles() {
36
+ if (!fs.existsSync(documentsPath)) fs.mkdirSync(documentsPath);
37
+ const liveSyncAvailable = await DocumentSyncQueue.enabled();
38
+ const directory = {
39
+ name: "documents",
40
+ type: "folder",
41
+ items: [],
42
+ };
43
+
44
+ for (const file of fs.readdirSync(documentsPath)) {
45
+ if (path.extname(file) === ".md") continue;
46
+ const folderPath = path.resolve(documentsPath, file);
47
+ const isFolder = fs.lstatSync(folderPath).isDirectory();
48
+ if (isFolder) {
49
+ const subdocs = {
50
+ name: file,
51
+ type: "folder",
52
+ items: [],
53
+ };
54
+
55
+ const subfiles = fs.readdirSync(folderPath);
56
+ const filenames = {};
57
+ const filePromises = [];
58
+
59
+ for (let i = 0; i < subfiles.length; i++) {
60
+ const subfile = subfiles[i];
61
+ const cachefilename = `${file}/${subfile}`;
62
+ if (path.extname(subfile) !== ".json") continue;
63
+ filePromises.push(
64
+ fileToPickerData({
65
+ pathToFile: path.join(folderPath, subfile),
66
+ liveSyncAvailable,
67
+ cachefilename,
68
+ })
69
+ );
70
+ filenames[cachefilename] = subfile;
71
+ }
72
+ const results = await Promise.all(filePromises)
73
+ .then((results) => results.filter((i) => !!i)) // Remove null results
74
+ .then((results) => results.filter((i) => hasRequiredMetadata(i))); // Remove invalid file structures
75
+ subdocs.items.push(...results);
76
+
77
+ // Grab the pinned workspaces and watched documents for this folder's documents
78
+ // at the time of the query so we don't have to re-query the database for each file
79
+ const pinnedWorkspacesByDocument =
80
+ await getPinnedWorkspacesByDocument(filenames);
81
+ const watchedDocumentsFilenames =
82
+ await getWatchedDocumentFilenames(filenames);
83
+ for (const item of subdocs.items) {
84
+ item.pinnedWorkspaces = pinnedWorkspacesByDocument[item.name] || [];
85
+ item.watched =
86
+ watchedDocumentsFilenames.hasOwnProperty(item.name) || false;
87
+ }
88
+
89
+ directory.items.push(subdocs);
90
+ }
91
+ }
92
+
93
+ // Make sure custom-documents is always the first folder in picker
94
+ directory.items = [
95
+ directory.items.find((folder) => folder.name === "custom-documents"),
96
+ ...directory.items.filter((folder) => folder.name !== "custom-documents"),
97
+ ].filter((i) => !!i);
98
+
99
+ return directory;
100
+ }
101
+
102
+ /**
103
+ * Gets the documents by folder name.
104
+ * @param {string} folderName - The name of the folder to get the documents from.
105
+ * @returns {Promise<{folder: string, documents: any[], code: number, error: string}>} - The documents by folder name.
106
+ */
107
+ async function getDocumentsByFolder(folderName = "") {
108
+ if (!folderName) {
109
+ return {
110
+ folder: folderName,
111
+ documents: [],
112
+ code: 400,
113
+ error: "Folder name must be provided.",
114
+ };
115
+ }
116
+
117
+ const folderPath = path.resolve(documentsPath, normalizePath(folderName));
118
+ if (
119
+ !isWithin(documentsPath, folderPath) ||
120
+ !fs.existsSync(folderPath) ||
121
+ !fs.lstatSync(folderPath).isDirectory()
122
+ ) {
123
+ return {
124
+ folder: folderName,
125
+ documents: [],
126
+ code: 404,
127
+ error: `Folder "${folderName}" does not exist.`,
128
+ };
129
+ }
130
+
131
+ const documents = [];
132
+ const filenames = {};
133
+ const files = fs.readdirSync(folderPath);
134
+ for (const file of files) {
135
+ if (path.extname(file) !== ".json") continue;
136
+ const filePath = path.join(folderPath, file);
137
+ const rawData = fs.readFileSync(filePath, "utf8");
138
+ const cachefilename = `${folderName}/${file}`;
139
+ const { pageContent: _pageContent, ...metadata } = JSON.parse(rawData);
140
+ documents.push({
141
+ name: file,
142
+ type: "file",
143
+ ...metadata,
144
+ cached: await cachedVectorInformation(cachefilename, true),
145
+ });
146
+ filenames[cachefilename] = file;
147
+ }
148
+
149
+ // Get pinned and watched information for each document in the folder
150
+ const pinnedWorkspacesByDocument =
151
+ await getPinnedWorkspacesByDocument(filenames);
152
+ const watchedDocumentsFilenames =
153
+ await getWatchedDocumentFilenames(filenames);
154
+ for (let doc of documents) {
155
+ doc.pinnedWorkspaces = pinnedWorkspacesByDocument[doc.name] || [];
156
+ doc.watched = Object.prototype.hasOwnProperty.call(
157
+ watchedDocumentsFilenames,
158
+ doc.name
159
+ );
160
+ }
161
+
162
+ return { folder: folderName, documents, code: 200, error: null };
163
+ }
164
+
165
+ /**
166
+ * Searches the vector-cache folder for existing information so we dont have to re-embed a
167
+ * document and can instead push directly to vector db.
168
+ * @param {string} filename - the filename to check for cached vector information
169
+ * @param {boolean} checkOnly - if true, only check if the file exists, do not return the cached data
170
+ * @returns {Promise<{exists: boolean, chunks: any[]}>} - a promise that resolves to an object containing the existence of the file and its cached chunks
171
+ */
172
+ async function cachedVectorInformation(filename = null, checkOnly = false) {
173
+ if (!filename) return checkOnly ? false : { exists: false, chunks: [] };
174
+
175
+ const digest = uuidv5(filename, uuidv5.URL);
176
+ const file = path.resolve(vectorCachePath, `${digest}.json`);
177
+ const exists = fs.existsSync(file);
178
+
179
+ if (checkOnly) return exists;
180
+ if (!exists) return { exists, chunks: [] };
181
+
182
+ console.log(
183
+ `Cached vectorized results of ${filename} found! Using cached data to save on embed costs.`
184
+ );
185
+ const rawData = fs.readFileSync(file, "utf8");
186
+ return { exists: true, chunks: JSON.parse(rawData) };
187
+ }
188
+
189
+ // vectorData: pre-chunked vectorized data for a given file that includes the proper metadata and chunk-size limit so it can be iterated and dumped into Pinecone, etc
190
+ // filename is the fullpath to the doc so we can compare by filename to find cached matches.
191
+ async function storeVectorResult(vectorData = [], filename = null) {
192
+ if (!filename) return;
193
+ console.log(
194
+ `Caching vectorized results of ${filename} to prevent duplicated embedding.`
195
+ );
196
+ if (!fs.existsSync(vectorCachePath)) fs.mkdirSync(vectorCachePath);
197
+
198
+ const digest = uuidv5(filename, uuidv5.URL);
199
+ const writeTo = path.resolve(vectorCachePath, `${digest}.json`);
200
+ fs.writeFileSync(writeTo, JSON.stringify(vectorData), "utf8");
201
+ return;
202
+ }
203
+
204
+ // Purges a file from the documents/ folder.
205
+ async function purgeSourceDocument(filename = null) {
206
+ if (!filename) return;
207
+ const filePath = path.resolve(documentsPath, normalizePath(filename));
208
+
209
+ if (
210
+ !fs.existsSync(filePath) ||
211
+ !isWithin(documentsPath, filePath) ||
212
+ !fs.lstatSync(filePath).isFile()
213
+ )
214
+ return;
215
+
216
+ console.log(`Purging source document of ${filename}.`);
217
+ fs.rmSync(filePath);
218
+ return;
219
+ }
220
+
221
+ // Purges a vector-cache file from the vector-cache/ folder.
222
+ async function purgeVectorCache(filename = null) {
223
+ if (!filename) return;
224
+ const digest = uuidv5(filename, uuidv5.URL);
225
+ const filePath = path.resolve(vectorCachePath, `${digest}.json`);
226
+
227
+ if (!fs.existsSync(filePath) || !fs.lstatSync(filePath).isFile()) return;
228
+ console.log(`Purging vector-cache of ${filename}.`);
229
+ fs.rmSync(filePath);
230
+ return;
231
+ }
232
+
233
+ // Search for a specific document by its unique name in the entire `documents`
234
+ // folder via iteration of all folders and checking if the expected file exists.
235
+ async function findDocumentInDocuments(documentName = null) {
236
+ if (!documentName) return null;
237
+ for (const folder of fs.readdirSync(documentsPath)) {
238
+ const isFolder = fs
239
+ .lstatSync(path.join(documentsPath, folder))
240
+ .isDirectory();
241
+ if (!isFolder) continue;
242
+
243
+ const targetFilename = normalizePath(documentName);
244
+ const targetFileLocation = path.join(documentsPath, folder, targetFilename);
245
+
246
+ if (
247
+ !fs.existsSync(targetFileLocation) ||
248
+ !isWithin(documentsPath, targetFileLocation)
249
+ )
250
+ continue;
251
+
252
+ const fileData = fs.readFileSync(targetFileLocation, "utf8");
253
+ const cachefilename = `${folder}/${targetFilename}`;
254
+ const { pageContent: _pageContent, ...metadata } = JSON.parse(fileData);
255
+ return {
256
+ name: targetFilename,
257
+ type: "file",
258
+ ...metadata,
259
+ cached: await cachedVectorInformation(cachefilename, true),
260
+ };
261
+ }
262
+
263
+ return null;
264
+ }
265
+
266
+ /**
267
+ * Checks if a given path is within another path.
268
+ * @param {string} outer - The outer path (should be resolved).
269
+ * @param {string} inner - The inner path (should be resolved).
270
+ * @returns {boolean} - Returns true if the inner path is within the outer path, false otherwise.
271
+ */
272
+ function isWithin(outer, inner) {
273
+ if (outer === inner) return false;
274
+ const rel = path.relative(outer, inner);
275
+ return !rel.startsWith("../") && rel !== "..";
276
+ }
277
+
278
+ function normalizePath(filepath = "") {
279
+ const result = path
280
+ .normalize(filepath.trim())
281
+ .replace(/^(\.\.(\/|\\|$))+/, "")
282
+ .trim();
283
+ if (["..", ".", "/"].includes(result)) throw new Error("Invalid path.");
284
+ return result;
285
+ }
286
+
287
+ /**
288
+ * Strips characters that are illegal in Windows filenames, including Unicode
289
+ * quotation marks (U+201C, U+201D, etc.) that can get corrupted into ASCII
290
+ * double-quotes during charset conversion in the upload pipeline.
291
+ * @param {string} fileName - The filename to sanitize.
292
+ * @returns {string} - The sanitized filename.
293
+ */
294
+ function sanitizeFileName(fileName) {
295
+ if (!fileName) return fileName;
296
+ return fileName.replace(
297
+ /[<>:"/\\|?*\u201C\u201D\u201E\u201F\u2018\u2019\u201A\u201B]/g,
298
+ ""
299
+ );
300
+ }
301
+
302
+ // Check if the vector-cache folder is empty or not
303
+ // useful for it the user is changing embedders as this will
304
+ // break the previous cache.
305
+ function hasVectorCachedFiles() {
306
+ try {
307
+ return (
308
+ fs.readdirSync(vectorCachePath)?.filter((name) => name.endsWith(".json"))
309
+ .length !== 0
310
+ );
311
+ } catch {}
312
+ return false;
313
+ }
314
+
315
+ /**
316
+ * @param {string[]} filenames - array of filenames to check for pinned workspaces
317
+ * @returns {Promise<Record<string, string[]>>} - a record of filenames and their corresponding workspaceIds
318
+ */
319
+ async function getPinnedWorkspacesByDocument(filenames = []) {
320
+ return (
321
+ await Document.where(
322
+ {
323
+ docpath: {
324
+ in: Object.keys(filenames),
325
+ },
326
+ pinned: true,
327
+ },
328
+ null,
329
+ null,
330
+ null,
331
+ {
332
+ workspaceId: true,
333
+ docpath: true,
334
+ }
335
+ )
336
+ ).reduce((result, { workspaceId, docpath }) => {
337
+ const filename = filenames[docpath];
338
+ if (!result[filename]) result[filename] = [];
339
+ if (!result[filename].includes(workspaceId))
340
+ result[filename].push(workspaceId);
341
+ return result;
342
+ }, {});
343
+ }
344
+
345
+ /**
346
+ * Get a record of filenames and their corresponding workspaceIds that have watched a document
347
+ * that will be used to determine if a document should be displayed in the watched documents sidebar
348
+ * @param {string[]} filenames - array of filenames to check for watched workspaces
349
+ * @returns {Promise<Record<string, string[]>>} - a record of filenames and their corresponding workspaceIds
350
+ */
351
+ async function getWatchedDocumentFilenames(filenames = []) {
352
+ return (
353
+ await Document.where(
354
+ {
355
+ docpath: { in: Object.keys(filenames) },
356
+ watched: true,
357
+ },
358
+ null,
359
+ null,
360
+ null,
361
+ { workspaceId: true, docpath: true }
362
+ )
363
+ ).reduce((result, { workspaceId, docpath }) => {
364
+ const filename = filenames[docpath];
365
+ result[filename] = workspaceId;
366
+ return result;
367
+ }, {});
368
+ }
369
+
370
+ /**
371
+ * Purges the entire vector-cache folder and recreates it.
372
+ * @returns {void}
373
+ */
374
+ function purgeEntireVectorCache() {
375
+ fs.rmSync(vectorCachePath, { recursive: true, force: true });
376
+ fs.mkdirSync(vectorCachePath);
377
+ return;
378
+ }
379
+
380
+ /**
381
+ * File size threshold for files that are too large to be read into memory (MB)
382
+ *
383
+ * If the file is larger than this, we will stream it and parse it in chunks
384
+ * This is to prevent us from using too much memory when parsing large files
385
+ * or loading the files in the file picker.
386
+ * @TODO - When lazy loading for folders is implemented, we should increase this threshold (512MB)
387
+ * since it will always be faster to readSync than to stream the file and parse it in chunks.
388
+ */
389
+ const FILE_READ_SIZE_THRESHOLD = 150 * (1024 * 1024);
390
+
391
+ /**
392
+ * Converts a file to picker data
393
+ * @param {string} pathToFile - The path to the file to convert
394
+ * @param {boolean} liveSyncAvailable - Whether live sync is available
395
+ * @returns {Promise<{name: string, type: string, [string]: any, cached: boolean, canWatch: boolean}>} - The picker data
396
+ */
397
+ async function fileToPickerData({
398
+ pathToFile,
399
+ liveSyncAvailable = false,
400
+ cachefilename = null,
401
+ }) {
402
+ let metadata = {};
403
+ const filename = path.basename(pathToFile);
404
+ const fileStats = fs.statSync(pathToFile);
405
+ const cachedStatus = await cachedVectorInformation(cachefilename, true);
406
+
407
+ if (fileStats.size < FILE_READ_SIZE_THRESHOLD) {
408
+ const rawData = fs.readFileSync(pathToFile, "utf8");
409
+ try {
410
+ metadata = JSON.parse(rawData);
411
+ // Remove the pageContent field from the metadata - it is large and not needed for the picker
412
+ delete metadata.pageContent;
413
+ } catch (err) {
414
+ console.error("Error parsing file", err);
415
+ return null;
416
+ }
417
+
418
+ return {
419
+ name: filename,
420
+ type: "file",
421
+ ...metadata,
422
+ cached: cachedStatus,
423
+ canWatch: liveSyncAvailable
424
+ ? DocumentSyncQueue.canWatch(metadata)
425
+ : false,
426
+ // pinnedWorkspaces: [], // This is the list of workspaceIds that have pinned this document
427
+ // watched: false, // boolean to indicate if this document is watched in ANY workspace
428
+ };
429
+ }
430
+
431
+ console.log(
432
+ `Stream-parsing ${path.basename(pathToFile)} because it exceeds the ${FILE_READ_SIZE_THRESHOLD} byte limit.`
433
+ );
434
+ const stream = fs.createReadStream(pathToFile, { encoding: "utf8" });
435
+ try {
436
+ let fileContent = "";
437
+ metadata = await new Promise((resolve, reject) => {
438
+ stream
439
+ .on("data", (chunk) => {
440
+ fileContent += chunk;
441
+ })
442
+ .on("end", () => {
443
+ metadata = JSON.parse(fileContent);
444
+ // Remove the pageContent field from the metadata - it is large and not needed for the picker
445
+ delete metadata.pageContent;
446
+ resolve(metadata);
447
+ })
448
+ .on("error", (err) => {
449
+ console.error("Error parsing file", err);
450
+ reject(null);
451
+ });
452
+ }).catch((err) => {
453
+ console.error("Error parsing file", err);
454
+ });
455
+ } catch (err) {
456
+ console.error("Error parsing file", err);
457
+ metadata = null;
458
+ } finally {
459
+ stream.destroy();
460
+ }
461
+
462
+ // If the metadata is empty or something went wrong, return null
463
+ if (!metadata || !Object.keys(metadata)?.length) {
464
+ console.log(`Stream-parsing failed for ${path.basename(pathToFile)}`);
465
+ return null;
466
+ }
467
+
468
+ return {
469
+ name: filename,
470
+ type: "file",
471
+ ...metadata,
472
+ cached: cachedStatus,
473
+ canWatch: liveSyncAvailable ? DocumentSyncQueue.canWatch(metadata) : false,
474
+ };
475
+ }
476
+
477
+ const REQUIRED_FILE_OBJECT_FIELDS = [
478
+ "name",
479
+ "type",
480
+ "url",
481
+ "title",
482
+ "docAuthor",
483
+ "description",
484
+ "docSource",
485
+ "chunkSource",
486
+ "published",
487
+ "wordCount",
488
+ "token_count_estimate",
489
+ ];
490
+
491
+ /**
492
+ * Checks if a given metadata object has all the required fields
493
+ * @param {{name: string, type: string, url: string, title: string, docAuthor: string, description: string, docSource: string, chunkSource: string, published: string, wordCount: number, token_count_estimate: number}} metadata - The metadata object to check (fileToPickerData)
494
+ * @returns {boolean} - Returns true if the metadata object has all the required fields, false otherwise
495
+ */
496
+ function hasRequiredMetadata(metadata = {}) {
497
+ return REQUIRED_FILE_OBJECT_FIELDS.every((field) =>
498
+ metadata.hasOwnProperty(field)
499
+ );
500
+ }
501
+
502
+ module.exports = {
503
+ findDocumentInDocuments,
504
+ cachedVectorInformation,
505
+ viewLocalFiles,
506
+ purgeSourceDocument,
507
+ purgeVectorCache,
508
+ storeVectorResult,
509
+ fileData,
510
+ normalizePath,
511
+ isWithin,
512
+ documentsPath,
513
+ directUploadsPath,
514
+ hasVectorCachedFiles,
515
+ purgeEntireVectorCache,
516
+ getDocumentsByFolder,
517
+ hotdirPath,
518
+ sanitizeFileName,
519
+ };