totem-llm 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +22 -0
- package/README.md +81 -0
- package/bin/totem-llm.js +80 -0
- package/collector/.env +9 -0
- package/collector/.env.example +9 -0
- package/collector/.nvmrc +1 -0
- package/collector/extensions/index.js +239 -0
- package/collector/extensions/resync/index.js +197 -0
- package/collector/hotdir/__HOTDIR__.md +3 -0
- package/collector/index.js +202 -0
- package/collector/middleware/httpLogger.js +29 -0
- package/collector/middleware/setDataSigner.js +44 -0
- package/collector/middleware/verifyIntegrity.js +32 -0
- package/collector/package.json +67 -0
- package/collector/processLink/convert/generic.js +240 -0
- package/collector/processLink/helpers/index.js +190 -0
- package/collector/processLink/index.js +47 -0
- package/collector/processRawText/index.js +83 -0
- package/collector/processSingleFile/convert/asAudio.js +79 -0
- package/collector/processSingleFile/convert/asDocx.js +63 -0
- package/collector/processSingleFile/convert/asEPub.js +61 -0
- package/collector/processSingleFile/convert/asImage.js +56 -0
- package/collector/processSingleFile/convert/asMbox.js +83 -0
- package/collector/processSingleFile/convert/asOfficeMime.js +59 -0
- package/collector/processSingleFile/convert/asPDF/PDFLoader/index.js +97 -0
- package/collector/processSingleFile/convert/asPDF/index.js +86 -0
- package/collector/processSingleFile/convert/asTxt.js +59 -0
- package/collector/processSingleFile/convert/asXlsx.js +193 -0
- package/collector/processSingleFile/index.js +95 -0
- package/collector/utils/EncryptionWorker/index.js +77 -0
- package/collector/utils/OCRLoader/index.js +355 -0
- package/collector/utils/OCRLoader/validLangs.js +155 -0
- package/collector/utils/WhisperProviders/OpenAiWhisper.js +49 -0
- package/collector/utils/WhisperProviders/ffmpeg/index.js +114 -0
- package/collector/utils/WhisperProviders/localWhisper.js +197 -0
- package/collector/utils/comKey/index.js +54 -0
- package/collector/utils/constants.js +92 -0
- package/collector/utils/downloadURIToFile/index.js +90 -0
- package/collector/utils/extensions/Confluence/ConfluenceLoader/index.js +154 -0
- package/collector/utils/extensions/Confluence/index.js +292 -0
- package/collector/utils/extensions/DrupalWiki/DrupalWiki/index.js +338 -0
- package/collector/utils/extensions/DrupalWiki/index.js +102 -0
- package/collector/utils/extensions/ObsidianVault/index.js +95 -0
- package/collector/utils/extensions/PaperlessNgx/PaperlessNgxLoader/index.js +157 -0
- package/collector/utils/extensions/PaperlessNgx/index.js +128 -0
- package/collector/utils/extensions/RepoLoader/GithubRepo/RepoLoader/index.js +265 -0
- package/collector/utils/extensions/RepoLoader/GithubRepo/index.js +159 -0
- package/collector/utils/extensions/RepoLoader/GitlabRepo/RepoLoader/index.js +447 -0
- package/collector/utils/extensions/RepoLoader/GitlabRepo/index.js +261 -0
- package/collector/utils/extensions/RepoLoader/index.js +41 -0
- package/collector/utils/extensions/WebsiteDepth/index.js +211 -0
- package/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/index.js +99 -0
- package/collector/utils/extensions/YoutubeTranscript/YoutubeLoader/youtube-transcript.js +246 -0
- package/collector/utils/extensions/YoutubeTranscript/index.js +219 -0
- package/collector/utils/files/index.js +240 -0
- package/collector/utils/files/mime.js +64 -0
- package/collector/utils/http/index.js +54 -0
- package/collector/utils/logger/index.js +68 -0
- package/collector/utils/runtimeSettings/index.js +97 -0
- package/collector/utils/shell.js +25 -0
- package/collector/utils/tokenizer/index.js +66 -0
- package/collector/utils/url/index.js +137 -0
- package/collector/yarn.lock +4413 -0
- package/lib/config.js +79 -0
- package/lib/launcher.js +116 -0
- package/lib/setup.js +143 -0
- package/package.json +64 -0
- package/scripts/postinstall.mjs +117 -0
- package/server/.env +489 -0
- package/server/.env.example +496 -0
- package/server/.flowconfig +30 -0
- package/server/.nvmrc +1 -0
- package/server/endpoints/admin.js +566 -0
- package/server/endpoints/agentFileServer.js +174 -0
- package/server/endpoints/agentFlows.js +192 -0
- package/server/endpoints/agentSkillWhitelist.js +82 -0
- package/server/endpoints/agentWebsocket.js +62 -0
- package/server/endpoints/api/admin/index.js +775 -0
- package/server/endpoints/api/auth/index.js +33 -0
- package/server/endpoints/api/document/index.js +1210 -0
- package/server/endpoints/api/embed/index.js +409 -0
- package/server/endpoints/api/index.js +29 -0
- package/server/endpoints/api/openai/compatibility-test-script.cjs +79 -0
- package/server/endpoints/api/openai/helpers.js +50 -0
- package/server/endpoints/api/openai/index.js +321 -0
- package/server/endpoints/api/system/index.js +275 -0
- package/server/endpoints/api/userManagement/index.js +124 -0
- package/server/endpoints/api/workspace/index.js +1008 -0
- package/server/endpoints/api/workspaceThread/index.js +627 -0
- package/server/endpoints/browserExtension.js +221 -0
- package/server/endpoints/chat.js +193 -0
- package/server/endpoints/communityHub.js +214 -0
- package/server/endpoints/document.js +111 -0
- package/server/endpoints/embed/index.js +103 -0
- package/server/endpoints/embedManagement.js +131 -0
- package/server/endpoints/experimental/imported-agent-plugins.js +65 -0
- package/server/endpoints/experimental/index.js +12 -0
- package/server/endpoints/experimental/liveSync.js +110 -0
- package/server/endpoints/extensions/index.js +175 -0
- package/server/endpoints/invite.js +81 -0
- package/server/endpoints/mcpServers.js +127 -0
- package/server/endpoints/memory.js +196 -0
- package/server/endpoints/mobile/index.js +160 -0
- package/server/endpoints/mobile/middleware/index.js +97 -0
- package/server/endpoints/mobile/utils/index.js +195 -0
- package/server/endpoints/modelRouter.js +205 -0
- package/server/endpoints/scheduledJobs.js +374 -0
- package/server/endpoints/system.js +1490 -0
- package/server/endpoints/telegram.js +333 -0
- package/server/endpoints/utils/dockerModelRunnerUtils.js +94 -0
- package/server/endpoints/utils/googleAgentSkillEndpoints.js +70 -0
- package/server/endpoints/utils/lemonadeUtilsEndpoints.js +170 -0
- package/server/endpoints/utils/outlookAgentUtils.js +190 -0
- package/server/endpoints/utils.js +220 -0
- package/server/endpoints/webPush.js +27 -0
- package/server/endpoints/workspaceThreads.js +245 -0
- package/server/endpoints/workspaces.js +1159 -0
- package/server/endpoints/workspacesParsedFiles.js +208 -0
- package/server/index.js +178 -0
- package/server/jobs/cleanup-generated-files.js +160 -0
- package/server/jobs/cleanup-orphan-documents.js +79 -0
- package/server/jobs/embedding-worker.js +197 -0
- package/server/jobs/extract-memories.js +192 -0
- package/server/jobs/handle-telegram-chat.js +67 -0
- package/server/jobs/helpers/index.js +54 -0
- package/server/jobs/helpers/memory-extraction-utils.js +366 -0
- package/server/jobs/helpers/scheduled-job-helper.js +111 -0
- package/server/jobs/run-scheduled-job.js +157 -0
- package/server/jobs/sync-watched-documents.js +209 -0
- package/server/jsconfig.json +14 -0
- package/server/middleware/httpLogger.js +23 -0
- package/server/models/agentSkillWhitelist.js +100 -0
- package/server/models/apiKeys.js +99 -0
- package/server/models/browserExtensionApiKey.js +190 -0
- package/server/models/cacheData.js +69 -0
- package/server/models/communityHub.js +213 -0
- package/server/models/documentSyncQueue.js +252 -0
- package/server/models/documentSyncRun.js +88 -0
- package/server/models/documents.js +356 -0
- package/server/models/embedChats.js +199 -0
- package/server/models/embedConfig.js +251 -0
- package/server/models/eventLogs.js +129 -0
- package/server/models/externalCommunicationConnector.js +110 -0
- package/server/models/invite.js +144 -0
- package/server/models/memory.js +451 -0
- package/server/models/mobileDevice.js +251 -0
- package/server/models/modelRouter.js +259 -0
- package/server/models/modelRouterRule.js +336 -0
- package/server/models/passwordRecovery.js +115 -0
- package/server/models/promptHistory.js +107 -0
- package/server/models/scheduledJob.js +499 -0
- package/server/models/scheduledJobRun.js +362 -0
- package/server/models/slashCommandsPresets.js +143 -0
- package/server/models/systemPromptVariables.js +361 -0
- package/server/models/systemSettings.js +1183 -0
- package/server/models/temporaryAuthToken.js +104 -0
- package/server/models/user.js +381 -0
- package/server/models/vectors.js +79 -0
- package/server/models/workspace.js +720 -0
- package/server/models/workspaceAgentInvocation.js +97 -0
- package/server/models/workspaceChats.js +386 -0
- package/server/models/workspaceParsedFiles.js +246 -0
- package/server/models/workspaceThread.js +151 -0
- package/server/models/workspaceUsers.js +103 -0
- package/server/models/workspacesSuggestedMessages.js +83 -0
- package/server/package.json +130 -0
- package/server/prisma/migrations/20230921191814_init/migration.sql +125 -0
- package/server/prisma/migrations/20231101001441_init/migration.sql +2 -0
- package/server/prisma/migrations/20231101195421_init/migration.sql +11 -0
- package/server/prisma/migrations/20231129012019_add/migration.sql +2 -0
- package/server/prisma/migrations/20240113013409_init/migration.sql +2 -0
- package/server/prisma/migrations/20240118201333_init/migration.sql +2 -0
- package/server/prisma/migrations/20240202002020_init/migration.sql +37 -0
- package/server/prisma/migrations/20240206181106_init/migration.sql +13 -0
- package/server/prisma/migrations/20240206211916_init/migration.sql +11 -0
- package/server/prisma/migrations/20240208224848_init/migration.sql +24 -0
- package/server/prisma/migrations/20240210004405_init/migration.sql +2 -0
- package/server/prisma/migrations/20240216214639_init/migration.sql +2 -0
- package/server/prisma/migrations/20240219211018_init/migration.sql +2 -0
- package/server/prisma/migrations/20240301002308_init/migration.sql +2 -0
- package/server/prisma/migrations/20240326231053_init/migration.sql +2 -0
- package/server/prisma/migrations/20240405015034_init/migration.sql +2 -0
- package/server/prisma/migrations/20240412183346_init/migration.sql +24 -0
- package/server/prisma/migrations/20240425004220_init/migration.sql +30 -0
- package/server/prisma/migrations/20240430230707_init/migration.sql +2 -0
- package/server/prisma/migrations/20240510032311_init/migration.sql +15 -0
- package/server/prisma/migrations/20240618224346_init/migration.sql +26 -0
- package/server/prisma/migrations/20240821215625_init/migration.sql +2 -0
- package/server/prisma/migrations/20240824005054_init/migration.sql +15 -0
- package/server/prisma/migrations/20241003192954_init/migration.sql +2 -0
- package/server/prisma/migrations/20241029203722_init/migration.sql +12 -0
- package/server/prisma/migrations/20241029233509_init/migration.sql +5 -0
- package/server/prisma/migrations/20250102204948_init/migration.sql +2 -0
- package/server/prisma/migrations/20250226005538_init/migration.sql +2 -0
- package/server/prisma/migrations/20250318154720_init/migration.sql +18 -0
- package/server/prisma/migrations/20250506214129_init/migration.sql +13 -0
- package/server/prisma/migrations/20250709230835_init/migration.sql +2 -0
- package/server/prisma/migrations/20250725194841_init/migration.sql +17 -0
- package/server/prisma/migrations/20250808171557_init/migration.sql +23 -0
- package/server/prisma/migrations/20260130040204_init/migration.sql +2 -0
- package/server/prisma/migrations/20260313192859_init/migration.sql +10 -0
- package/server/prisma/migrations/20260319202916_init/migration.sql +12 -0
- package/server/prisma/migrations/20260406120000_init/migration.sql +2 -0
- package/server/prisma/migrations/20260423191158_init/migration.sql +29 -0
- package/server/prisma/migrations/20260424013043_init/migration.sql +22 -0
- package/server/prisma/migrations/20260520160208_init/migration.sql +46 -0
- package/server/prisma/migrations/migration_lock.toml +3 -0
- package/server/prisma/schema.prisma +484 -0
- package/server/prisma/seed.js +31 -0
- package/server/public/KaTeX_AMS-Regular.eot +0 -0
- package/server/public/KaTeX_AMS-Regular.ttf +0 -0
- package/server/public/KaTeX_AMS-Regular.woff +0 -0
- package/server/public/KaTeX_AMS-Regular.woff2 +0 -0
- package/server/public/KaTeX_Caligraphic-Bold.eot +0 -0
- package/server/public/KaTeX_Caligraphic-Bold.ttf +0 -0
- package/server/public/KaTeX_Caligraphic-Bold.woff +0 -0
- package/server/public/KaTeX_Caligraphic-Bold.woff2 +0 -0
- package/server/public/KaTeX_Caligraphic-Regular.eot +0 -0
- package/server/public/KaTeX_Caligraphic-Regular.ttf +0 -0
- package/server/public/KaTeX_Caligraphic-Regular.woff +0 -0
- package/server/public/KaTeX_Caligraphic-Regular.woff2 +0 -0
- package/server/public/KaTeX_Fraktur-Bold.eot +0 -0
- package/server/public/KaTeX_Fraktur-Bold.ttf +0 -0
- package/server/public/KaTeX_Fraktur-Bold.woff +0 -0
- package/server/public/KaTeX_Fraktur-Bold.woff2 +0 -0
- package/server/public/KaTeX_Fraktur-Regular.eot +0 -0
- package/server/public/KaTeX_Fraktur-Regular.ttf +0 -0
- package/server/public/KaTeX_Fraktur-Regular.woff +0 -0
- package/server/public/KaTeX_Fraktur-Regular.woff2 +0 -0
- package/server/public/KaTeX_Main-Bold.eot +0 -0
- package/server/public/KaTeX_Main-Bold.ttf +0 -0
- package/server/public/KaTeX_Main-Bold.woff +0 -0
- package/server/public/KaTeX_Main-Bold.woff2 +0 -0
- package/server/public/KaTeX_Main-Italic.eot +0 -0
- package/server/public/KaTeX_Main-Italic.ttf +0 -0
- package/server/public/KaTeX_Main-Italic.woff +0 -0
- package/server/public/KaTeX_Main-Italic.woff2 +0 -0
- package/server/public/KaTeX_Main-Regular.eot +0 -0
- package/server/public/KaTeX_Main-Regular.ttf +0 -0
- package/server/public/KaTeX_Main-Regular.woff +0 -0
- package/server/public/KaTeX_Main-Regular.woff2 +0 -0
- package/server/public/KaTeX_Math-Italic.eot +0 -0
- package/server/public/KaTeX_Math-Italic.ttf +0 -0
- package/server/public/KaTeX_Math-Italic.woff +0 -0
- package/server/public/KaTeX_Math-Italic.woff2 +0 -0
- package/server/public/KaTeX_SansSerif-Regular.eot +0 -0
- package/server/public/KaTeX_SansSerif-Regular.ttf +0 -0
- package/server/public/KaTeX_SansSerif-Regular.woff +0 -0
- package/server/public/KaTeX_SansSerif-Regular.woff2 +0 -0
- package/server/public/KaTeX_Script-Regular.eot +0 -0
- package/server/public/KaTeX_Script-Regular.ttf +0 -0
- package/server/public/KaTeX_Script-Regular.woff +0 -0
- package/server/public/KaTeX_Script-Regular.woff2 +0 -0
- package/server/public/KaTeX_Size1-Regular.eot +0 -0
- package/server/public/KaTeX_Size1-Regular.ttf +0 -0
- package/server/public/KaTeX_Size1-Regular.woff +0 -0
- package/server/public/KaTeX_Size1-Regular.woff2 +0 -0
- package/server/public/KaTeX_Size2-Regular.eot +0 -0
- package/server/public/KaTeX_Size2-Regular.ttf +0 -0
- package/server/public/KaTeX_Size2-Regular.woff +0 -0
- package/server/public/KaTeX_Size2-Regular.woff2 +0 -0
- package/server/public/KaTeX_Size3-Regular.eot +0 -0
- package/server/public/KaTeX_Size3-Regular.ttf +0 -0
- package/server/public/KaTeX_Size3-Regular.woff +0 -0
- package/server/public/KaTeX_Size4-Regular.eot +0 -0
- package/server/public/KaTeX_Size4-Regular.ttf +0 -0
- package/server/public/KaTeX_Size4-Regular.woff +0 -0
- package/server/public/KaTeX_Size4-Regular.woff2 +0 -0
- package/server/public/KaTeX_Typewriter-Regular.eot +0 -0
- package/server/public/KaTeX_Typewriter-Regular.ttf +0 -0
- package/server/public/KaTeX_Typewriter-Regular.woff +0 -0
- package/server/public/KaTeX_Typewriter-Regular.woff2 +0 -0
- package/server/public/PlusJakartaSans.ttf +0 -0
- package/server/public/_index.html +45 -0
- package/server/public/agent-animation.webm +0 -0
- package/server/public/agent-static.png +0 -0
- package/server/public/anthropic.png +0 -0
- package/server/public/anything-llm-dark.png +0 -0
- package/server/public/anything-llm-light.png +0 -0
- package/server/public/anything-llm.png +0 -0
- package/server/public/apipie.png +0 -0
- package/server/public/assets/404-aca8e9d6.js +1 -0
- package/server/public/assets/Brain-24e386d5.js +1 -0
- package/server/public/assets/Browser-1820fad0.js +1 -0
- package/server/public/assets/ChatText-35275002.js +1 -0
- package/server/public/assets/Check-7e83fb0d.js +1 -0
- package/server/public/assets/Copy-d323e545.js +1 -0
- package/server/public/assets/Database-f1a76329.js +1 -0
- package/server/public/assets/Eye-9f033dc0.js +1 -0
- package/server/public/assets/File-b92df2f3.js +1 -0
- package/server/public/assets/Gauge-e6762f4d.js +1 -0
- package/server/public/assets/Hammer-05b5ccc7.js +1 -0
- package/server/public/assets/Plug-7d951c7d.js +1 -0
- package/server/public/assets/PlusCircle-31c6a8d8.js +1 -0
- package/server/public/assets/RunDetailPage-edfdbd4d.js +1 -0
- package/server/public/assets/RunHistoryPage-732dae26.js +1 -0
- package/server/public/assets/Stop-4b47ae05.js +1 -0
- package/server/public/assets/Trans-36b04c7b.js +1 -0
- package/server/public/assets/WarningCircle-27467274.js +1 -0
- package/server/public/assets/XCircle-edfe554c.js +1 -0
- package/server/public/assets/admin-04b020fe.js +1 -0
- package/server/public/assets/agentFlows-e4124dd3.js +1 -0
- package/server/public/assets/anything-llm-infinity-0d7244a1.js +1 -0
- package/server/public/assets/communityHub-aba61686.js +1 -0
- package/server/public/assets/cron-2ad9194f.js +1 -0
- package/server/public/assets/files-493643b0.js +23 -0
- package/server/public/assets/github-dark-57c241de.js +1 -0
- package/server/public/assets/index-08cd0bc2.js +9 -0
- package/server/public/assets/index-0bf8ea25.js +1 -0
- package/server/public/assets/index-25b2660c.js +1 -0
- package/server/public/assets/index-2a7c0b04.js +1 -0
- package/server/public/assets/index-3643e990.js +1 -0
- package/server/public/assets/index-3761d55d.js +1 -0
- package/server/public/assets/index-4b5e1c76.js +5 -0
- package/server/public/assets/index-4fb4d421.js +2 -0
- package/server/public/assets/index-56bb11a0.js +1 -0
- package/server/public/assets/index-5c320a5d.js +1 -0
- package/server/public/assets/index-62160652.js +3 -0
- package/server/public/assets/index-62fee0f9.js +4 -0
- package/server/public/assets/index-65801ac6.js +1 -0
- package/server/public/assets/index-71027183.js +1 -0
- package/server/public/assets/index-72832304.js +18 -0
- package/server/public/assets/index-7b0ea06b.js +14 -0
- package/server/public/assets/index-8132c765.js +1 -0
- package/server/public/assets/index-837868b3.js +1 -0
- package/server/public/assets/index-86e8a6b4.js +1 -0
- package/server/public/assets/index-8ba96f42.js +1 -0
- package/server/public/assets/index-8e43159f.js +1 -0
- package/server/public/assets/index-92574f3a.js +1 -0
- package/server/public/assets/index-925792fe.js +2 -0
- package/server/public/assets/index-92d3b7bc.js +1 -0
- package/server/public/assets/index-994c849a.js +1 -0
- package/server/public/assets/index-9f71774b.js +1 -0
- package/server/public/assets/index-9f7dabd6.js +1 -0
- package/server/public/assets/index-a0e2be61.js +1 -0
- package/server/public/assets/index-a6ac5511.js +1 -0
- package/server/public/assets/index-a6c0d98a.js +1 -0
- package/server/public/assets/index-aa687cfe.js +3 -0
- package/server/public/assets/index-ab35fb74.js +1 -0
- package/server/public/assets/index-ab72b353.js +4 -0
- package/server/public/assets/index-ad8e77d2.js +5 -0
- package/server/public/assets/index-c07cc087.js +110 -0
- package/server/public/assets/index-c22f7da1.js +1 -0
- package/server/public/assets/index-c9b59c35.js +1 -0
- package/server/public/assets/index-d59c6d7f.js +1 -0
- package/server/public/assets/index-d69f2e09.js +1 -0
- package/server/public/assets/index-e5b911ea.js +1 -0
- package/server/public/assets/index-e6c540e1.js +1 -0
- package/server/public/assets/index-f86e167c.js +4 -0
- package/server/public/assets/index-f984ef26.js +1 -0
- package/server/public/assets/index-fb26b60a.js +1 -0
- package/server/public/assets/index-fd922432.js +1 -0
- package/server/public/assets/main-8ad4162f.js +1 -0
- package/server/public/assets/mcpServers-957b1f1c.js +1 -0
- package/server/public/assets/moment-3ad2e1a6.js +10 -0
- package/server/public/assets/ort.min-faca3c2b.js +1827 -0
- package/server/public/assets/piper-o91UDS6e-0c78e94e.js +4 -0
- package/server/public/assets/purify-6b86d6dc.js +26 -0
- package/server/public/assets/react-beautiful-dnd.esm-86e87aba.js +68 -0
- package/server/public/assets/truncate-352eb33b.js +1 -0
- package/server/public/assets/types-e90e1137.js +1 -0
- package/server/public/assets/usePolling-417bb099.js +1 -0
- package/server/public/assets/voices_static-D_OtJDHM-5cdb0950.js +1 -0
- package/server/public/assets/worker-8f433525.js +1 -0
- package/server/public/azure.png +0 -0
- package/server/public/baidu.png +0 -0
- package/server/public/bedrock.png +0 -0
- package/server/public/bg.png +0 -0
- package/server/public/bing.png +0 -0
- package/server/public/cerebras.png +0 -0
- package/server/public/cohere.png +0 -0
- package/server/public/cometapi.png +0 -0
- package/server/public/community-hub.png +0 -0
- package/server/public/confluence.jpeg +0 -0
- package/server/public/deepseek.png +0 -0
- package/server/public/docker-model-runner.png +0 -0
- package/server/public/dpais.png +0 -0
- package/server/public/drupalwiki.jpg +0 -0
- package/server/public/duckduckgo.png +0 -0
- package/server/public/elevenlabs.png +0 -0
- package/server/public/embed/anythingllm-chat-widget.min.css +1 -0
- package/server/public/embed/anythingllm-chat-widget.min.js +37 -0
- package/server/public/exa.png +0 -0
- package/server/public/favicon.ico +0 -0
- package/server/public/favicon.png +0 -0
- package/server/public/file-system.png +0 -0
- package/server/public/files.css +1 -0
- package/server/public/fonts/PlusJakartaSans.ttf +0 -0
- package/server/public/foundry-local.png +0 -0
- package/server/public/gemini.png +0 -0
- package/server/public/generate-charts.png +0 -0
- package/server/public/generate-save-files.png +0 -0
- package/server/public/generic-openai.png +0 -0
- package/server/public/giteeai.png +0 -0
- package/server/public/github-dark.css +10 -0
- package/server/public/github.css +19 -0
- package/server/public/github.svg +4 -0
- package/server/public/gitlab.svg +7 -0
- package/server/public/gmail.png +0 -0
- package/server/public/google-calendar.png +0 -0
- package/server/public/gplay-badge.svg +55 -0
- package/server/public/huggingface.png +0 -0
- package/server/public/index.css +1 -0
- package/server/public/index.js +1739 -0
- package/server/public/koboldcpp.png +0 -0
- package/server/public/lancedb.png +0 -0
- package/server/public/lemonade.png +0 -0
- package/server/public/link.svg +1 -0
- package/server/public/litellm.png +0 -0
- package/server/public/lmstudio.png +0 -0
- package/server/public/localai.png +0 -0
- package/server/public/login-logo-light.svg +45 -0
- package/server/public/login-logo.svg +45 -0
- package/server/public/manifest.json +13 -0
- package/server/public/mcp-logo.svg +1 -0
- package/server/public/milvus.png +0 -0
- package/server/public/minimax.png +0 -0
- package/server/public/mistral.jpeg +0 -0
- package/server/public/moonshotai.png +0 -0
- package/server/public/mssql.png +0 -0
- package/server/public/mysql.png +0 -0
- package/server/public/nvidia-nim.png +0 -0
- package/server/public/obsidian.png +0 -0
- package/server/public/ollama.png +0 -0
- package/server/public/openai.png +0 -0
- package/server/public/openrouter.jpeg +0 -0
- package/server/public/outlook.png +0 -0
- package/server/public/paperless-ngx.jpeg +0 -0
- package/server/public/perplexity.png +0 -0
- package/server/public/pgvector.png +0 -0
- package/server/public/piper.png +0 -0
- package/server/public/privatemode.png +0 -0
- package/server/public/qdrant.png +0 -0
- package/server/public/rag-memory.png +0 -0
- package/server/public/robots.txt +2 -0
- package/server/public/router-animation.webm +0 -0
- package/server/public/sambanova.png +0 -0
- package/server/public/scrape-websites.png +0 -0
- package/server/public/searchapi.png +0 -0
- package/server/public/searxng.png +0 -0
- package/server/public/serpapi.png +0 -0
- package/server/public/serper.png +0 -0
- package/server/public/serply.png +0 -0
- package/server/public/service-workers/push-notifications.js +26 -0
- package/server/public/sql-agent.png +0 -0
- package/server/public/tavily.svg +1 -0
- package/server/public/text-generation-webui.png +0 -0
- package/server/public/thinking-animation.webm +0 -0
- package/server/public/thinking-static.png +0 -0
- package/server/public/togetherai.png +0 -0
- package/server/public/view-summarize.png +0 -0
- package/server/public/voyageai.png +0 -0
- package/server/public/weaviate.png +0 -0
- package/server/public/xai.png +0 -0
- package/server/public/youtube.svg +10 -0
- package/server/public/zai.png +0 -0
- package/server/public/zilliz.png +0 -0
- package/server/utils/AiProviders/anthropic/index.js +372 -0
- package/server/utils/AiProviders/apipie/index.js +386 -0
- package/server/utils/AiProviders/azureOpenAi/index.js +232 -0
- package/server/utils/AiProviders/bedrock/index.js +787 -0
- package/server/utils/AiProviders/bedrock/utils.js +166 -0
- package/server/utils/AiProviders/cerebras/index.js +314 -0
- package/server/utils/AiProviders/cohere/index.js +251 -0
- package/server/utils/AiProviders/cometapi/constants.js +39 -0
- package/server/utils/AiProviders/cometapi/index.js +445 -0
- package/server/utils/AiProviders/deepseek/index.js +180 -0
- package/server/utils/AiProviders/dellProAiStudio/index.js +217 -0
- package/server/utils/AiProviders/dockerModelRunner/index.js +520 -0
- package/server/utils/AiProviders/fireworksAi/index.js +273 -0
- package/server/utils/AiProviders/foundry/index.js +475 -0
- package/server/utils/AiProviders/gemini/defaultModels.js +74 -0
- package/server/utils/AiProviders/gemini/index.js +457 -0
- package/server/utils/AiProviders/gemini/syncStaticLists.mjs +48 -0
- package/server/utils/AiProviders/genericOpenAi/index.js +485 -0
- package/server/utils/AiProviders/giteeai/index.js +245 -0
- package/server/utils/AiProviders/groq/index.js +257 -0
- package/server/utils/AiProviders/huggingface/index.js +165 -0
- package/server/utils/AiProviders/koboldCPP/index.js +264 -0
- package/server/utils/AiProviders/lemonade/index.js +463 -0
- package/server/utils/AiProviders/liteLLM/index.js +201 -0
- package/server/utils/AiProviders/lmStudio/index.js +383 -0
- package/server/utils/AiProviders/localAi/index.js +198 -0
- package/server/utils/AiProviders/minimax/index.js +165 -0
- package/server/utils/AiProviders/mistral/index.js +191 -0
- package/server/utils/AiProviders/modelMap/index.js +212 -0
- package/server/utils/AiProviders/modelMap/legacy.js +162 -0
- package/server/utils/AiProviders/modelRouter/index.js +170 -0
- package/server/utils/AiProviders/moonshotAi/index.js +176 -0
- package/server/utils/AiProviders/novita/index.js +460 -0
- package/server/utils/AiProviders/nvidiaNim/index.js +253 -0
- package/server/utils/AiProviders/ollama/index.js +516 -0
- package/server/utils/AiProviders/openAi/index.js +301 -0
- package/server/utils/AiProviders/openRouter/index.js +567 -0
- package/server/utils/AiProviders/perplexity/index.js +304 -0
- package/server/utils/AiProviders/perplexity/models.js +24 -0
- package/server/utils/AiProviders/perplexity/scripts/chat_models.txt +6 -0
- package/server/utils/AiProviders/perplexity/scripts/parse.mjs +49 -0
- package/server/utils/AiProviders/ppio/index.js +274 -0
- package/server/utils/AiProviders/privatemode/index.js +220 -0
- package/server/utils/AiProviders/sambanova/index.js +273 -0
- package/server/utils/AiProviders/textGenWebUI/index.js +197 -0
- package/server/utils/AiProviders/togetherAi/index.js +263 -0
- package/server/utils/AiProviders/xai/index.js +200 -0
- package/server/utils/AiProviders/zai/index.js +184 -0
- package/server/utils/BackgroundWorkers/index.js +432 -0
- package/server/utils/DocumentManager/index.js +72 -0
- package/server/utils/EmbeddingEngines/azureOpenAi/index.js +115 -0
- package/server/utils/EmbeddingEngines/cohere/index.js +91 -0
- package/server/utils/EmbeddingEngines/gemini/index.js +136 -0
- package/server/utils/EmbeddingEngines/genericOpenAi/index.js +128 -0
- package/server/utils/EmbeddingEngines/lemonade/index.js +99 -0
- package/server/utils/EmbeddingEngines/liteLLM/index.js +102 -0
- package/server/utils/EmbeddingEngines/lmstudio/index.js +122 -0
- package/server/utils/EmbeddingEngines/localAi/index.js +122 -0
- package/server/utils/EmbeddingEngines/mistral/index.js +40 -0
- package/server/utils/EmbeddingEngines/native/constants.js +63 -0
- package/server/utils/EmbeddingEngines/native/index.js +290 -0
- package/server/utils/EmbeddingEngines/ollama/index.js +136 -0
- package/server/utils/EmbeddingEngines/openAi/index.js +102 -0
- package/server/utils/EmbeddingEngines/openRouter/index.js +131 -0
- package/server/utils/EmbeddingEngines/voyageAi/index.js +71 -0
- package/server/utils/EmbeddingRerankers/native/index.js +257 -0
- package/server/utils/EmbeddingWorkerManager.js +202 -0
- package/server/utils/EncryptionManager/index.js +85 -0
- package/server/utils/MCP/hypervisor/index.js +554 -0
- package/server/utils/MCP/index.js +269 -0
- package/server/utils/PasswordRecovery/index.js +103 -0
- package/server/utils/PushNotifications/index.js +228 -0
- package/server/utils/TextSplitter/index.js +206 -0
- package/server/utils/TextToSpeech/elevenLabs/index.js +54 -0
- package/server/utils/TextToSpeech/index.js +18 -0
- package/server/utils/TextToSpeech/openAi/index.js +29 -0
- package/server/utils/TextToSpeech/openAiGeneric/index.js +58 -0
- package/server/utils/agentFlows/executor.js +233 -0
- package/server/utils/agentFlows/executors/api-call.js +60 -0
- package/server/utils/agentFlows/executors/llm-instruction.js +47 -0
- package/server/utils/agentFlows/executors/web-scraping.js +110 -0
- package/server/utils/agentFlows/flowTypes.js +85 -0
- package/server/utils/agentFlows/index.js +288 -0
- package/server/utils/agents/aibitat/error.js +18 -0
- package/server/utils/agents/aibitat/example/beginner-chat.js +56 -0
- package/server/utils/agents/aibitat/example/blog-post-coding.js +55 -0
- package/server/utils/agents/aibitat/example/websocket/index.html +67 -0
- package/server/utils/agents/aibitat/example/websocket/websock-branding-collab.js +100 -0
- package/server/utils/agents/aibitat/example/websocket/websock-multi-turn-chat.js +91 -0
- package/server/utils/agents/aibitat/index.js +1437 -0
- package/server/utils/agents/aibitat/plugins/chat-history.js +210 -0
- package/server/utils/agents/aibitat/plugins/cli.js +134 -0
- package/server/utils/agents/aibitat/plugins/create-files/assets/anything-llm-invert.png +0 -0
- package/server/utils/agents/aibitat/plugins/create-files/assets/anything-llm.png +0 -0
- package/server/utils/agents/aibitat/plugins/create-files/docx/create-docx-file.js +301 -0
- package/server/utils/agents/aibitat/plugins/create-files/docx/test-themes.js +298 -0
- package/server/utils/agents/aibitat/plugins/create-files/docx/utils.js +1095 -0
- package/server/utils/agents/aibitat/plugins/create-files/index.js +23 -0
- package/server/utils/agents/aibitat/plugins/create-files/lib.js +300 -0
- package/server/utils/agents/aibitat/plugins/create-files/pdf/create-pdf-file.js +138 -0
- package/server/utils/agents/aibitat/plugins/create-files/pdf/utils.js +70 -0
- package/server/utils/agents/aibitat/plugins/create-files/pptx/create-presentation.js +342 -0
- package/server/utils/agents/aibitat/plugins/create-files/pptx/section-agent.js +257 -0
- package/server/utils/agents/aibitat/plugins/create-files/pptx/test-themes.js +143 -0
- package/server/utils/agents/aibitat/plugins/create-files/pptx/themes.js +181 -0
- package/server/utils/agents/aibitat/plugins/create-files/pptx/utils.js +378 -0
- package/server/utils/agents/aibitat/plugins/create-files/text/create-text-file.js +158 -0
- package/server/utils/agents/aibitat/plugins/create-files/xlsx/create-excel-file.js +362 -0
- package/server/utils/agents/aibitat/plugins/create-files/xlsx/utils.js +333 -0
- package/server/utils/agents/aibitat/plugins/file-history.js +37 -0
- package/server/utils/agents/aibitat/plugins/filesystem/copy-file.js +127 -0
- package/server/utils/agents/aibitat/plugins/filesystem/create-directory.js +84 -0
- package/server/utils/agents/aibitat/plugins/filesystem/edit-file.js +133 -0
- package/server/utils/agents/aibitat/plugins/filesystem/get-file-info.js +75 -0
- package/server/utils/agents/aibitat/plugins/filesystem/index.js +33 -0
- package/server/utils/agents/aibitat/plugins/filesystem/lib.js +787 -0
- package/server/utils/agents/aibitat/plugins/filesystem/list-directory.js +175 -0
- package/server/utils/agents/aibitat/plugins/filesystem/move-file.js +95 -0
- package/server/utils/agents/aibitat/plugins/filesystem/read-multiple-files.js +157 -0
- package/server/utils/agents/aibitat/plugins/filesystem/read-text-file.js +142 -0
- package/server/utils/agents/aibitat/plugins/filesystem/search-files.js +469 -0
- package/server/utils/agents/aibitat/plugins/filesystem/write-text-file.js +90 -0
- package/server/utils/agents/aibitat/plugins/gmail/account/gmail-get-mailbox-stats.js +75 -0
- package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-create-draft-reply.js +220 -0
- package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-create-draft.js +217 -0
- package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-delete-draft.js +87 -0
- package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-get-draft.js +84 -0
- package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-list-drafts.js +96 -0
- package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-send-draft.js +94 -0
- package/server/utils/agents/aibitat/plugins/gmail/drafts/gmail-update-draft.js +217 -0
- package/server/utils/agents/aibitat/plugins/gmail/index.js +73 -0
- package/server/utils/agents/aibitat/plugins/gmail/lib.js +573 -0
- package/server/utils/agents/aibitat/plugins/gmail/search/gmail-get-inbox.js +104 -0
- package/server/utils/agents/aibitat/plugins/gmail/search/gmail-read-thread.js +122 -0
- package/server/utils/agents/aibitat/plugins/gmail/search/gmail-search.js +121 -0
- package/server/utils/agents/aibitat/plugins/gmail/send/gmail-reply-to-thread.js +238 -0
- package/server/utils/agents/aibitat/plugins/gmail/send/gmail-send-email.js +242 -0
- package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-mark-read.js +87 -0
- package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-mark-unread.js +87 -0
- package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-move-to-archive.js +89 -0
- package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-move-to-inbox.js +89 -0
- package/server/utils/agents/aibitat/plugins/gmail/threads/gmail-move-to-trash.js +89 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/calendars/gcal-get-calendar.js +79 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/calendars/gcal-list-calendars.js +82 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-create-event.js +233 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-get-event.js +116 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-get-events-for-day.js +117 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-get-events.js +159 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-get-upcoming-events.js +249 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-quick-add.js +114 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-set-my-status.js +125 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/events/gcal-update-event.js +173 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/index.js +40 -0
- package/server/utils/agents/aibitat/plugins/google-calendar/lib.js +288 -0
- package/server/utils/agents/aibitat/plugins/http-socket.js +249 -0
- package/server/utils/agents/aibitat/plugins/index.js +47 -0
- package/server/utils/agents/aibitat/plugins/memory.js +167 -0
- package/server/utils/agents/aibitat/plugins/outlook/account/outlook-get-mailbox-stats.js +78 -0
- package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-create-draft.js +232 -0
- package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-delete-draft.js +86 -0
- package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-list-drafts.js +80 -0
- package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-send-draft.js +82 -0
- package/server/utils/agents/aibitat/plugins/outlook/drafts/outlook-update-draft.js +132 -0
- package/server/utils/agents/aibitat/plugins/outlook/index.js +45 -0
- package/server/utils/agents/aibitat/plugins/outlook/lib.js +1413 -0
- package/server/utils/agents/aibitat/plugins/outlook/search/outlook-get-inbox.js +75 -0
- package/server/utils/agents/aibitat/plugins/outlook/search/outlook-read-thread.js +131 -0
- package/server/utils/agents/aibitat/plugins/outlook/search/outlook-search.js +101 -0
- package/server/utils/agents/aibitat/plugins/outlook/send/outlook-send-email.js +222 -0
- package/server/utils/agents/aibitat/plugins/rechart.js +114 -0
- package/server/utils/agents/aibitat/plugins/request-user-input.js +265 -0
- package/server/utils/agents/aibitat/plugins/router-classifier.js +122 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MSSQL.js +114 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/MySQL.js +83 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/Postgresql.js +75 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/index.js +80 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/SQLConnectors/utils.js +182 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/get-table-schema.js +115 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/index.js +21 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/list-database.js +49 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/list-table.js +105 -0
- package/server/utils/agents/aibitat/plugins/sql-agent/query.js +101 -0
- package/server/utils/agents/aibitat/plugins/summarize.js +189 -0
- package/server/utils/agents/aibitat/plugins/web-browsing.js +1182 -0
- package/server/utils/agents/aibitat/plugins/web-scraping.js +158 -0
- package/server/utils/agents/aibitat/plugins/websocket.js +361 -0
- package/server/utils/agents/aibitat/providers/ai-provider.js +706 -0
- package/server/utils/agents/aibitat/providers/anthropic.js +490 -0
- package/server/utils/agents/aibitat/providers/apipie.js +99 -0
- package/server/utils/agents/aibitat/providers/azure.js +122 -0
- package/server/utils/agents/aibitat/providers/bedrock.js +456 -0
- package/server/utils/agents/aibitat/providers/cerebras.js +208 -0
- package/server/utils/agents/aibitat/providers/cohere.js +269 -0
- package/server/utils/agents/aibitat/providers/cometapi.js +104 -0
- package/server/utils/agents/aibitat/providers/deepseek.js +223 -0
- package/server/utils/agents/aibitat/providers/dellProAiStudio.js +106 -0
- package/server/utils/agents/aibitat/providers/dockerModelRunner.js +189 -0
- package/server/utils/agents/aibitat/providers/fireworksai.js +101 -0
- package/server/utils/agents/aibitat/providers/foundry.js +112 -0
- package/server/utils/agents/aibitat/providers/gemini.js +450 -0
- package/server/utils/agents/aibitat/providers/genericOpenAi.js +206 -0
- package/server/utils/agents/aibitat/providers/giteeai.js +94 -0
- package/server/utils/agents/aibitat/providers/groq.js +184 -0
- package/server/utils/agents/aibitat/providers/helpers/classes.js +16 -0
- package/server/utils/agents/aibitat/providers/helpers/tooled.js +384 -0
- package/server/utils/agents/aibitat/providers/helpers/untooled.js +437 -0
- package/server/utils/agents/aibitat/providers/index.js +77 -0
- package/server/utils/agents/aibitat/providers/koboldcpp.js +103 -0
- package/server/utils/agents/aibitat/providers/lemonade.js +213 -0
- package/server/utils/agents/aibitat/providers/litellm.js +186 -0
- package/server/utils/agents/aibitat/providers/lmstudio.js +195 -0
- package/server/utils/agents/aibitat/providers/localai.js +190 -0
- package/server/utils/agents/aibitat/providers/minimax.js +194 -0
- package/server/utils/agents/aibitat/providers/mistral.js +103 -0
- package/server/utils/agents/aibitat/providers/moonshotAi.js +97 -0
- package/server/utils/agents/aibitat/providers/novita.js +188 -0
- package/server/utils/agents/aibitat/providers/nvidiaNim.js +101 -0
- package/server/utils/agents/aibitat/providers/ollama.js +606 -0
- package/server/utils/agents/aibitat/providers/openai.js +345 -0
- package/server/utils/agents/aibitat/providers/openrouter.js +195 -0
- package/server/utils/agents/aibitat/providers/perplexity.js +99 -0
- package/server/utils/agents/aibitat/providers/ppio.js +104 -0
- package/server/utils/agents/aibitat/providers/privatemode.js +107 -0
- package/server/utils/agents/aibitat/providers/sambanova.js +94 -0
- package/server/utils/agents/aibitat/providers/textgenwebui.js +99 -0
- package/server/utils/agents/aibitat/providers/togetherai.js +100 -0
- package/server/utils/agents/aibitat/providers/xai.js +99 -0
- package/server/utils/agents/aibitat/providers/zai.js +97 -0
- package/server/utils/agents/aibitat/utils/dedupe.js +152 -0
- package/server/utils/agents/aibitat/utils/summarize.js +63 -0
- package/server/utils/agents/aibitat/utils/toolReranker.js +225 -0
- package/server/utils/agents/defaults.js +196 -0
- package/server/utils/agents/ephemeral.js +790 -0
- package/server/utils/agents/imported-manifest.schema.json +158 -0
- package/server/utils/agents/imported.js +317 -0
- package/server/utils/agents/index.js +883 -0
- package/server/utils/boot/MetaGenerator.js +376 -0
- package/server/utils/boot/eagerLoadContextWindows.js +40 -0
- package/server/utils/boot/index.js +90 -0
- package/server/utils/boot/markOnboarded.js +52 -0
- package/server/utils/chats/agents.js +111 -0
- package/server/utils/chats/apiChatHandler.js +864 -0
- package/server/utils/chats/commands/reset.js +37 -0
- package/server/utils/chats/embed.js +301 -0
- package/server/utils/chats/index.js +128 -0
- package/server/utils/chats/openaiCompatible.js +537 -0
- package/server/utils/chats/stream.js +376 -0
- package/server/utils/collectorApi/index.js +319 -0
- package/server/utils/comKey/index.js +86 -0
- package/server/utils/database/index.js +83 -0
- package/server/utils/files/index.js +519 -0
- package/server/utils/files/logo.js +117 -0
- package/server/utils/files/multer.js +178 -0
- package/server/utils/files/pfp.js +63 -0
- package/server/utils/files/purgeDocument.js +91 -0
- package/server/utils/helpers/admin/index.js +56 -0
- package/server/utils/helpers/agents.js +35 -0
- package/server/utils/helpers/camelcase.js +143 -0
- package/server/utils/helpers/chat/LLMPerformanceMonitor.js +113 -0
- package/server/utils/helpers/chat/convertTo.js +256 -0
- package/server/utils/helpers/chat/index.js +448 -0
- package/server/utils/helpers/chat/responses.js +383 -0
- package/server/utils/helpers/customModels.js +1142 -0
- package/server/utils/helpers/index.js +716 -0
- package/server/utils/helpers/portAvailabilityChecker.js +46 -0
- package/server/utils/helpers/search.js +94 -0
- package/server/utils/helpers/shell.js +25 -0
- package/server/utils/helpers/tiktoken.js +107 -0
- package/server/utils/helpers/updateENV.js +1399 -0
- package/server/utils/http/index.js +141 -0
- package/server/utils/logger/index.js +66 -0
- package/server/utils/memories/index.js +139 -0
- package/server/utils/middleware/chatHistoryViewable.js +18 -0
- package/server/utils/middleware/communityHubDownloadsEnabled.js +77 -0
- package/server/utils/middleware/embedMiddleware.js +178 -0
- package/server/utils/middleware/featureFlagEnabled.js +24 -0
- package/server/utils/middleware/isSupportedRepoProviders.js +12 -0
- package/server/utils/middleware/multiUserProtected.js +107 -0
- package/server/utils/middleware/requireFeature.js +46 -0
- package/server/utils/middleware/simpleSSOEnabled.js +84 -0
- package/server/utils/middleware/validApiKey.js +29 -0
- package/server/utils/middleware/validBrowserExtensionApiKey.js +51 -0
- package/server/utils/middleware/validWorkspace.js +52 -0
- package/server/utils/middleware/validatedRequest.js +111 -0
- package/server/utils/middleware/workspaceDeletionProtection.js +18 -0
- package/server/utils/prisma/PRISMA.md +47 -0
- package/server/utils/prisma/index.js +13 -0
- package/server/utils/router/index.js +653 -0
- package/server/utils/telegramBot/chat/agent.js +425 -0
- package/server/utils/telegramBot/chat/stream.js +471 -0
- package/server/utils/telegramBot/constants.js +23 -0
- package/server/utils/telegramBot/index.js +844 -0
- package/server/utils/telegramBot/utils/commands/handlers/handleAbort.js +15 -0
- package/server/utils/telegramBot/utils/commands/handlers/handleHelp.js +15 -0
- package/server/utils/telegramBot/utils/commands/handlers/handleHistory.js +80 -0
- package/server/utils/telegramBot/utils/commands/handlers/handleNewThread.js +35 -0
- package/server/utils/telegramBot/utils/commands/handlers/handleProof.js +202 -0
- package/server/utils/telegramBot/utils/commands/handlers/handleReset.js +32 -0
- package/server/utils/telegramBot/utils/commands/handlers/handleStart.js +19 -0
- package/server/utils/telegramBot/utils/commands/handlers/handleStatus.js +80 -0
- package/server/utils/telegramBot/utils/commands/handlers/showModelMenu.js +98 -0
- package/server/utils/telegramBot/utils/commands/handlers/showThreadMenu.js +120 -0
- package/server/utils/telegramBot/utils/commands/handlers/showWorkspaceMenu.js +84 -0
- package/server/utils/telegramBot/utils/commands/index.js +125 -0
- package/server/utils/telegramBot/utils/format.js +225 -0
- package/server/utils/telegramBot/utils/index.js +203 -0
- package/server/utils/telegramBot/utils/media.js +163 -0
- package/server/utils/telegramBot/utils/messageQueue.js +53 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleBackSources.js +16 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleBackWorkspaces.js +18 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleModelCancel.js +14 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleModelPagination.js +24 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleModelSelect.js +52 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleSourcePagination.js +24 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleSourceSelect.js +106 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleThreadPagination.js +26 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleThreadSelect.js +43 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleToolApproval.js +104 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleWorkspaceCreate.js +33 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleWorkspacePagination.js +26 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/handleWorkspaceSelect.js +24 -0
- package/server/utils/telegramBot/utils/navigation/callbacks/index.js +59 -0
- package/server/utils/telegramBot/utils/navigation/index.js +36 -0
- package/server/utils/telegramBot/utils/verification.js +171 -0
- package/server/utils/vectorDbProviders/astra/ASTRA_SETUP.md +22 -0
- package/server/utils/vectorDbProviders/astra/index.js +474 -0
- package/server/utils/vectorDbProviders/base.js +202 -0
- package/server/utils/vectorDbProviders/chroma/index.js +484 -0
- package/server/utils/vectorDbProviders/chromacloud/index.js +158 -0
- package/server/utils/vectorDbProviders/lance/index.js +506 -0
- package/server/utils/vectorDbProviders/milvus/MILVUS_SETUP.md +40 -0
- package/server/utils/vectorDbProviders/milvus/index.js +434 -0
- package/server/utils/vectorDbProviders/pgvector/SETUP.md +125 -0
- package/server/utils/vectorDbProviders/pgvector/index.js +850 -0
- package/server/utils/vectorDbProviders/pinecone/PINECONE_SETUP.md +24 -0
- package/server/utils/vectorDbProviders/pinecone/index.js +317 -0
- package/server/utils/vectorDbProviders/qdrant/QDRANT_SETUP.md +17 -0
- package/server/utils/vectorDbProviders/qdrant/index.js +442 -0
- package/server/utils/vectorDbProviders/weaviate/WEAVIATE_SETUP.md +17 -0
- package/server/utils/vectorDbProviders/weaviate/index.js +510 -0
- package/server/utils/vectorDbProviders/zilliz/index.js +36 -0
- package/server/utils/vectorStore/resetAllVectorStores.js +59 -0
- package/server/yarn.lock +10702 -0
|
@@ -0,0 +1,787 @@
|
|
|
1
|
+
const fs = require("fs/promises");
|
|
2
|
+
const path = require("path");
|
|
3
|
+
const os = require("os");
|
|
4
|
+
const { randomBytes } = require("crypto");
|
|
5
|
+
const { createTwoFilesPatch } = require("diff");
|
|
6
|
+
const { humanFileSize } = require("../../../../helpers");
|
|
7
|
+
|
|
8
|
+
/**
|
|
9
|
+
* Manages filesystem operations with security constraints.
|
|
10
|
+
* Ensures all file operations stay within allowed directories.
|
|
11
|
+
*/
|
|
12
|
+
class FilesystemManager {
|
|
13
|
+
static FILE_READ_CHUNK_SIZE = 1024;
|
|
14
|
+
static CONTEXT_RESERVE_RATIO = 0.25;
|
|
15
|
+
static IMAGE_EXTENSIONS = [
|
|
16
|
+
".png",
|
|
17
|
+
".jpg",
|
|
18
|
+
".jpeg",
|
|
19
|
+
".gif",
|
|
20
|
+
".webp",
|
|
21
|
+
".svg",
|
|
22
|
+
".bmp",
|
|
23
|
+
];
|
|
24
|
+
static IMAGE_MIME_TYPES = {
|
|
25
|
+
".png": "image/png",
|
|
26
|
+
".jpg": "image/jpeg",
|
|
27
|
+
".jpeg": "image/jpeg",
|
|
28
|
+
".gif": "image/gif",
|
|
29
|
+
".webp": "image/webp",
|
|
30
|
+
".svg": "image/svg+xml",
|
|
31
|
+
".bmp": "image/bmp",
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
/**
|
|
35
|
+
* Checks if the filesystem tool is available.
|
|
36
|
+
* The filesystem tool is only available when running in a docker container
|
|
37
|
+
* or in development mode.
|
|
38
|
+
* @returns {boolean} True if the tool is available
|
|
39
|
+
*/
|
|
40
|
+
isToolAvailable() {
|
|
41
|
+
if (process.env.NODE_ENV === "development") return true;
|
|
42
|
+
return process.env.ANYTHING_LLM_RUNTIME === "docker";
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
#allowedDirectories = [];
|
|
46
|
+
#isInitialized = false;
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Gets the default filesystem root path.
|
|
50
|
+
* @returns {string} The default filesystem root path
|
|
51
|
+
*/
|
|
52
|
+
#getDefaultFilesystemRoot() {
|
|
53
|
+
const storageRoot =
|
|
54
|
+
process.env.STORAGE_DIR ||
|
|
55
|
+
path.resolve(__dirname, "../../../../../storage");
|
|
56
|
+
return path.join(storageRoot, "anythingllm-fs");
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* Initializes the filesystem with default or configured directories.
|
|
61
|
+
* @param {string[]} [directories] - Optional array of directories to allow
|
|
62
|
+
* @returns {Promise<string[]>} The initialized allowed directories
|
|
63
|
+
*/
|
|
64
|
+
async #initializeFilesystem(directories = null) {
|
|
65
|
+
if (directories && directories.length > 0) {
|
|
66
|
+
this.#allowedDirectories = directories.map((dir) =>
|
|
67
|
+
path.resolve(this.#expandHome(dir))
|
|
68
|
+
);
|
|
69
|
+
} else {
|
|
70
|
+
const defaultRoot = this.#getDefaultFilesystemRoot();
|
|
71
|
+
this.#allowedDirectories = [defaultRoot];
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
for (const dir of this.#allowedDirectories) {
|
|
75
|
+
try {
|
|
76
|
+
await fs.mkdir(dir, { recursive: true });
|
|
77
|
+
} catch (error) {
|
|
78
|
+
console.error(
|
|
79
|
+
`Warning: Could not create directory ${dir}: ${error.message}`
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
this.#isInitialized = true;
|
|
85
|
+
return this.#allowedDirectories;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* Expands home directory tildes in paths.
|
|
90
|
+
* @param {string} filepath - The path to expand
|
|
91
|
+
* @returns {string} Expanded path
|
|
92
|
+
*/
|
|
93
|
+
#expandHome(filepath) {
|
|
94
|
+
if (filepath.startsWith("~/") || filepath === "~") {
|
|
95
|
+
return path.join(os.homedir(), filepath.slice(1));
|
|
96
|
+
}
|
|
97
|
+
return filepath;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
/**
|
|
101
|
+
* Normalizes a path by standardizing format.
|
|
102
|
+
* @param {string} p - The path to normalize
|
|
103
|
+
* @returns {string} Normalized path
|
|
104
|
+
*/
|
|
105
|
+
#normalizePath(p) {
|
|
106
|
+
p = p.trim().replace(/^["']|["']$/g, "");
|
|
107
|
+
if (p.startsWith("/"))
|
|
108
|
+
return p.replace(/\/+/g, "/").replace(/(?<!^)\/$/, "");
|
|
109
|
+
return path.normalize(p);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
/**
|
|
113
|
+
* Validates and normalizes a path string for security checks.
|
|
114
|
+
* @param {string} p - The path to validate
|
|
115
|
+
* @returns {string|null} Normalized absolute path or null if invalid
|
|
116
|
+
*/
|
|
117
|
+
#normalizeAndValidatePath(p) {
|
|
118
|
+
if (typeof p !== "string" || !p || p.includes("\x00")) return null;
|
|
119
|
+
try {
|
|
120
|
+
const normalized = path.resolve(path.normalize(p));
|
|
121
|
+
return path.isAbsolute(normalized) ? normalized : null;
|
|
122
|
+
} catch {
|
|
123
|
+
return null;
|
|
124
|
+
}
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Checks if an absolute path is within any of the allowed directories.
|
|
129
|
+
* @param {string} absolutePath - The absolute path to check
|
|
130
|
+
* @param {string[]} allowedDirs - Array of allowed directory paths
|
|
131
|
+
* @returns {boolean} True if path is within allowed directories
|
|
132
|
+
*/
|
|
133
|
+
#isPathWithinAllowedDirectories(absolutePath, allowedDirs) {
|
|
134
|
+
if (!Array.isArray(allowedDirs) || allowedDirs.length === 0) return false;
|
|
135
|
+
|
|
136
|
+
const normalizedPath = this.#normalizeAndValidatePath(absolutePath);
|
|
137
|
+
if (!normalizedPath) return false;
|
|
138
|
+
|
|
139
|
+
return allowedDirs.some((dir) => {
|
|
140
|
+
const normalizedDir = this.#normalizeAndValidatePath(dir);
|
|
141
|
+
if (!normalizedDir) return false;
|
|
142
|
+
|
|
143
|
+
if (normalizedPath === normalizedDir) return true;
|
|
144
|
+
return normalizedPath.startsWith(normalizedDir + path.sep);
|
|
145
|
+
});
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
/**
|
|
149
|
+
* Resolves a relative path against allowed directories.
|
|
150
|
+
* @param {string} relativePath - The relative path to resolve
|
|
151
|
+
* @returns {string} The resolved absolute path
|
|
152
|
+
*/
|
|
153
|
+
#resolveRelativePathAgainstAllowedDirectories(relativePath) {
|
|
154
|
+
if (this.#allowedDirectories.length === 0) {
|
|
155
|
+
return path.resolve(process.cwd(), relativePath);
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
for (const allowedDir of this.#allowedDirectories) {
|
|
159
|
+
const candidate = path.resolve(allowedDir, relativePath);
|
|
160
|
+
const normalizedCandidate = this.#normalizePath(candidate);
|
|
161
|
+
|
|
162
|
+
if (
|
|
163
|
+
this.#isPathWithinAllowedDirectories(
|
|
164
|
+
normalizedCandidate,
|
|
165
|
+
this.#allowedDirectories
|
|
166
|
+
)
|
|
167
|
+
) {
|
|
168
|
+
return candidate;
|
|
169
|
+
}
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return path.resolve(this.#allowedDirectories[0], relativePath);
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
/**
|
|
176
|
+
* Normalizes line endings to Unix-style.
|
|
177
|
+
* @param {string} text - Text to normalize
|
|
178
|
+
* @returns {string} Text with normalized line endings
|
|
179
|
+
*/
|
|
180
|
+
#normalizeLineEndings(text) {
|
|
181
|
+
return text.replace(/\r\n/g, "\n");
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* Writes content to a file atomically using a temp file and rename.
|
|
186
|
+
* @param {string} filePath - Path to the file
|
|
187
|
+
* @param {string} content - Content to write
|
|
188
|
+
* @returns {Promise<void>}
|
|
189
|
+
*/
|
|
190
|
+
async #atomicWrite(filePath, content) {
|
|
191
|
+
const tempPath = `${filePath}.${randomBytes(16).toString("hex")}.tmp`;
|
|
192
|
+
try {
|
|
193
|
+
await fs.writeFile(tempPath, content, "utf-8");
|
|
194
|
+
await fs.rename(tempPath, filePath);
|
|
195
|
+
} catch (error) {
|
|
196
|
+
try {
|
|
197
|
+
await fs.unlink(tempPath);
|
|
198
|
+
} catch {}
|
|
199
|
+
throw error;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
/**
|
|
204
|
+
* Reads file content with collector API fallback.
|
|
205
|
+
* @param {string} filePath - Path to the file
|
|
206
|
+
* @param {Function} processContent - Function to process parsed content
|
|
207
|
+
* @param {Function} rawFallback - Function to call if collector fails
|
|
208
|
+
* @returns {Promise<string>} Processed content
|
|
209
|
+
*/
|
|
210
|
+
async #withCollectorFallback(filePath, processContent, rawFallback) {
|
|
211
|
+
const parseResult = await this.#parseFileWithCollector(filePath);
|
|
212
|
+
if (parseResult.parsed && parseResult.content) {
|
|
213
|
+
return processContent(parseResult.content);
|
|
214
|
+
}
|
|
215
|
+
return rawFallback();
|
|
216
|
+
}
|
|
217
|
+
|
|
218
|
+
/**
|
|
219
|
+
* Creates a unified diff between two strings.
|
|
220
|
+
* @param {string} originalContent - Original content
|
|
221
|
+
* @param {string} newContent - New content
|
|
222
|
+
* @param {string} filepath - File path for diff header
|
|
223
|
+
* @returns {string} Unified diff string
|
|
224
|
+
*/
|
|
225
|
+
#createUnifiedDiff(originalContent, newContent, filepath = "file") {
|
|
226
|
+
const normalizedOriginal = this.#normalizeLineEndings(originalContent);
|
|
227
|
+
const normalizedNew = this.#normalizeLineEndings(newContent);
|
|
228
|
+
|
|
229
|
+
return createTwoFilesPatch(
|
|
230
|
+
filepath,
|
|
231
|
+
filepath,
|
|
232
|
+
normalizedOriginal,
|
|
233
|
+
normalizedNew,
|
|
234
|
+
"original",
|
|
235
|
+
"modified"
|
|
236
|
+
);
|
|
237
|
+
}
|
|
238
|
+
|
|
239
|
+
/**
|
|
240
|
+
* Reads file content as text using direct file read.
|
|
241
|
+
* @param {string} filePath - Path to the file
|
|
242
|
+
* @param {string} encoding - File encoding
|
|
243
|
+
* @returns {Promise<string>} File content
|
|
244
|
+
*/
|
|
245
|
+
async #readFileContentRaw(filePath, encoding = "utf-8") {
|
|
246
|
+
return await fs.readFile(filePath, encoding);
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
/**
|
|
250
|
+
* Parses a file using the collector API to extract text content.
|
|
251
|
+
* @param {string} filePath - Absolute path to the file
|
|
252
|
+
* @returns {Promise<{content: string, parsed: boolean}>} Parsed content
|
|
253
|
+
*/
|
|
254
|
+
async #parseFileWithCollector(filePath) {
|
|
255
|
+
try {
|
|
256
|
+
const { CollectorApi } = require("../../../../collectorApi");
|
|
257
|
+
const collectorApi = new CollectorApi();
|
|
258
|
+
|
|
259
|
+
const isOnline = await collectorApi.online();
|
|
260
|
+
if (!isOnline) {
|
|
261
|
+
return {
|
|
262
|
+
content: null,
|
|
263
|
+
parsed: false,
|
|
264
|
+
error: "Collector service offline",
|
|
265
|
+
};
|
|
266
|
+
}
|
|
267
|
+
|
|
268
|
+
const filename = path.basename(filePath);
|
|
269
|
+
const result = await collectorApi.parseDocument(filename, {
|
|
270
|
+
absolutePath: filePath,
|
|
271
|
+
});
|
|
272
|
+
|
|
273
|
+
if (!result || !result.success) {
|
|
274
|
+
return {
|
|
275
|
+
content: null,
|
|
276
|
+
parsed: false,
|
|
277
|
+
error: result?.reason || "Failed to parse document",
|
|
278
|
+
};
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
if (result.content) return { content: result.content, parsed: true };
|
|
282
|
+
if (result.documents && result.documents.length > 0) {
|
|
283
|
+
const content = result.documents
|
|
284
|
+
.map((doc) => doc.pageContent || doc.content || "")
|
|
285
|
+
.filter(Boolean)
|
|
286
|
+
.join("\n\n");
|
|
287
|
+
if (content) return { content, parsed: true };
|
|
288
|
+
}
|
|
289
|
+
|
|
290
|
+
return { content: null, parsed: false, error: "No content in response" };
|
|
291
|
+
} catch (error) {
|
|
292
|
+
return { content: null, parsed: false, error: error.message };
|
|
293
|
+
}
|
|
294
|
+
}
|
|
295
|
+
|
|
296
|
+
/**
|
|
297
|
+
* Gets the last N lines of a file using raw file operations.
|
|
298
|
+
* @param {string} filePath - Path to the file
|
|
299
|
+
* @param {number} numLines - Number of lines to return
|
|
300
|
+
* @returns {Promise<string>} Last N lines of the file
|
|
301
|
+
*/
|
|
302
|
+
async #tailFileRaw(filePath, numLines) {
|
|
303
|
+
const stats = await fs.stat(filePath);
|
|
304
|
+
const fileSize = stats.size;
|
|
305
|
+
|
|
306
|
+
if (fileSize === 0) return "";
|
|
307
|
+
|
|
308
|
+
const fileHandle = await fs.open(filePath, "r");
|
|
309
|
+
try {
|
|
310
|
+
const lines = [];
|
|
311
|
+
let position = fileSize;
|
|
312
|
+
const chunk = Buffer.alloc(FilesystemManager.FILE_READ_CHUNK_SIZE);
|
|
313
|
+
let linesFound = 0;
|
|
314
|
+
let remainingText = "";
|
|
315
|
+
|
|
316
|
+
while (position > 0 && linesFound < numLines) {
|
|
317
|
+
const size = Math.min(FilesystemManager.FILE_READ_CHUNK_SIZE, position);
|
|
318
|
+
position -= size;
|
|
319
|
+
|
|
320
|
+
const { bytesRead } = await fileHandle.read(chunk, 0, size, position);
|
|
321
|
+
if (!bytesRead) break;
|
|
322
|
+
|
|
323
|
+
const readData = chunk.slice(0, bytesRead).toString("utf-8");
|
|
324
|
+
const chunkText = readData + remainingText;
|
|
325
|
+
|
|
326
|
+
const chunkLines = this.#normalizeLineEndings(chunkText).split("\n");
|
|
327
|
+
|
|
328
|
+
if (position > 0) {
|
|
329
|
+
remainingText = chunkLines[0];
|
|
330
|
+
chunkLines.shift();
|
|
331
|
+
}
|
|
332
|
+
|
|
333
|
+
for (
|
|
334
|
+
let i = chunkLines.length - 1;
|
|
335
|
+
i >= 0 && linesFound < numLines;
|
|
336
|
+
i--
|
|
337
|
+
) {
|
|
338
|
+
lines.unshift(chunkLines[i]);
|
|
339
|
+
linesFound++;
|
|
340
|
+
}
|
|
341
|
+
}
|
|
342
|
+
|
|
343
|
+
return lines.join("\n");
|
|
344
|
+
} finally {
|
|
345
|
+
await fileHandle.close();
|
|
346
|
+
}
|
|
347
|
+
}
|
|
348
|
+
|
|
349
|
+
/**
|
|
350
|
+
* Gets the first N lines of a file using raw file operations.
|
|
351
|
+
* @param {string} filePath - Path to the file
|
|
352
|
+
* @param {number} numLines - Number of lines to return
|
|
353
|
+
* @returns {Promise<string>} First N lines of the file
|
|
354
|
+
*/
|
|
355
|
+
async #headFileRaw(filePath, numLines) {
|
|
356
|
+
const fileHandle = await fs.open(filePath, "r");
|
|
357
|
+
try {
|
|
358
|
+
const lines = [];
|
|
359
|
+
let buffer = "";
|
|
360
|
+
let bytesRead = 0;
|
|
361
|
+
const chunk = Buffer.alloc(FilesystemManager.FILE_READ_CHUNK_SIZE);
|
|
362
|
+
|
|
363
|
+
while (lines.length < numLines) {
|
|
364
|
+
const result = await fileHandle.read(chunk, 0, chunk.length, bytesRead);
|
|
365
|
+
if (result.bytesRead === 0) break;
|
|
366
|
+
bytesRead += result.bytesRead;
|
|
367
|
+
buffer += chunk.slice(0, result.bytesRead).toString("utf-8");
|
|
368
|
+
|
|
369
|
+
const newLineIndex = buffer.lastIndexOf("\n");
|
|
370
|
+
if (newLineIndex !== -1) {
|
|
371
|
+
const completeLines = buffer.slice(0, newLineIndex).split("\n");
|
|
372
|
+
buffer = buffer.slice(newLineIndex + 1);
|
|
373
|
+
for (const line of completeLines) {
|
|
374
|
+
lines.push(line);
|
|
375
|
+
if (lines.length >= numLines) break;
|
|
376
|
+
}
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
if (buffer.length > 0 && lines.length < numLines) {
|
|
381
|
+
lines.push(buffer);
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
return lines.join("\n");
|
|
385
|
+
} finally {
|
|
386
|
+
await fileHandle.close();
|
|
387
|
+
}
|
|
388
|
+
}
|
|
389
|
+
|
|
390
|
+
/**
|
|
391
|
+
* Gets the current allowed directories.
|
|
392
|
+
* @returns {string[]} Array of allowed directory paths
|
|
393
|
+
*/
|
|
394
|
+
getAllowedDirectories() {
|
|
395
|
+
return [...this.#allowedDirectories];
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
/**
|
|
399
|
+
* Ensures the filesystem is initialized before use.
|
|
400
|
+
* @returns {Promise<void>}
|
|
401
|
+
*/
|
|
402
|
+
async ensureInitialized() {
|
|
403
|
+
if (!this.#isInitialized) await this.#initializeFilesystem();
|
|
404
|
+
}
|
|
405
|
+
|
|
406
|
+
/**
|
|
407
|
+
* Validates a path for security, ensuring it's within allowed directories.
|
|
408
|
+
* @param {string} requestedPath - The path to validate
|
|
409
|
+
* @returns {Promise<string>} The validated absolute path
|
|
410
|
+
* @throws {Error} If path is outside allowed directories
|
|
411
|
+
*/
|
|
412
|
+
async validatePath(requestedPath) {
|
|
413
|
+
await this.ensureInitialized();
|
|
414
|
+
const expandedPath = this.#expandHome(requestedPath);
|
|
415
|
+
const absolute = path.isAbsolute(expandedPath)
|
|
416
|
+
? path.resolve(expandedPath)
|
|
417
|
+
: this.#resolveRelativePathAgainstAllowedDirectories(expandedPath);
|
|
418
|
+
|
|
419
|
+
const normalizedRequested = this.#normalizePath(absolute);
|
|
420
|
+
|
|
421
|
+
const isAllowed = this.#isPathWithinAllowedDirectories(
|
|
422
|
+
normalizedRequested,
|
|
423
|
+
this.#allowedDirectories
|
|
424
|
+
);
|
|
425
|
+
if (!isAllowed) {
|
|
426
|
+
console.log(
|
|
427
|
+
`[validatePath] Access denied - path outside allowed directories: ${absolute} not in ${this.#allowedDirectories.join(", ")}`
|
|
428
|
+
);
|
|
429
|
+
throw new Error(`Access denied - path outside allowed directories.`);
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
try {
|
|
433
|
+
const realPath = await fs.realpath(absolute);
|
|
434
|
+
const normalizedReal = this.#normalizePath(realPath);
|
|
435
|
+
if (
|
|
436
|
+
!this.#isPathWithinAllowedDirectories(
|
|
437
|
+
normalizedReal,
|
|
438
|
+
this.#allowedDirectories
|
|
439
|
+
)
|
|
440
|
+
) {
|
|
441
|
+
console.log(
|
|
442
|
+
`[validatePath] Access denied - symlink target outside allowed directories: ${realPath} not in ${this.#allowedDirectories.join(", ")}`
|
|
443
|
+
);
|
|
444
|
+
throw new Error(
|
|
445
|
+
`Access denied - symlink target outside allowed directories.`
|
|
446
|
+
);
|
|
447
|
+
}
|
|
448
|
+
return realPath;
|
|
449
|
+
} catch (error) {
|
|
450
|
+
if (error.code === "ENOENT") {
|
|
451
|
+
const parentDir = path.dirname(absolute);
|
|
452
|
+
try {
|
|
453
|
+
const realParentPath = await fs.realpath(parentDir);
|
|
454
|
+
const normalizedParent = this.#normalizePath(realParentPath);
|
|
455
|
+
if (
|
|
456
|
+
!this.#isPathWithinAllowedDirectories(
|
|
457
|
+
normalizedParent,
|
|
458
|
+
this.#allowedDirectories
|
|
459
|
+
)
|
|
460
|
+
) {
|
|
461
|
+
console.log(
|
|
462
|
+
`[validatePath] Access denied - parent directory outside allowed directories: ${realParentPath} not in ${this.#allowedDirectories.join(", ")}`
|
|
463
|
+
);
|
|
464
|
+
throw new Error(
|
|
465
|
+
`Access denied - parent directory outside allowed directories.`
|
|
466
|
+
);
|
|
467
|
+
}
|
|
468
|
+
return absolute;
|
|
469
|
+
} catch {
|
|
470
|
+
throw new Error(`Parent directory does not exist: ${parentDir}`);
|
|
471
|
+
}
|
|
472
|
+
}
|
|
473
|
+
throw error;
|
|
474
|
+
}
|
|
475
|
+
}
|
|
476
|
+
|
|
477
|
+
/**
|
|
478
|
+
* Gets detailed file statistics.
|
|
479
|
+
* @param {string} filePath - Path to the file
|
|
480
|
+
* @returns {Promise<Object>} File statistics
|
|
481
|
+
*/
|
|
482
|
+
async getFileStats(filePath) {
|
|
483
|
+
const stats = await fs.stat(filePath);
|
|
484
|
+
return {
|
|
485
|
+
size: stats.size,
|
|
486
|
+
sizeFormatted: humanFileSize(stats.size, true, 2),
|
|
487
|
+
created: stats.birthtime.toISOString(),
|
|
488
|
+
modified: stats.mtime.toISOString(),
|
|
489
|
+
accessed: stats.atime.toISOString(),
|
|
490
|
+
isDirectory: stats.isDirectory(),
|
|
491
|
+
isFile: stats.isFile(),
|
|
492
|
+
permissions: stats.mode.toString(8).slice(-3),
|
|
493
|
+
};
|
|
494
|
+
}
|
|
495
|
+
|
|
496
|
+
/**
|
|
497
|
+
* Reads file content, using the collector API to parse binary files.
|
|
498
|
+
* @param {string} filePath - Path to the file
|
|
499
|
+
* @param {string} encoding - File encoding (default: utf-8)
|
|
500
|
+
* @returns {Promise<string>} File content
|
|
501
|
+
*/
|
|
502
|
+
async readFileContent(filePath, encoding = "utf-8") {
|
|
503
|
+
return this.#withCollectorFallback(
|
|
504
|
+
filePath,
|
|
505
|
+
(content) => content,
|
|
506
|
+
() => this.#readFileContentRaw(filePath, encoding)
|
|
507
|
+
);
|
|
508
|
+
}
|
|
509
|
+
|
|
510
|
+
/**
|
|
511
|
+
* Writes content to a file securely.
|
|
512
|
+
* @param {string} filePath - Path to the file
|
|
513
|
+
* @param {string} content - Content to write
|
|
514
|
+
* @returns {Promise<void>}
|
|
515
|
+
*/
|
|
516
|
+
async writeFileContent(filePath, content) {
|
|
517
|
+
try {
|
|
518
|
+
await fs.writeFile(filePath, content, { encoding: "utf-8", flag: "wx" });
|
|
519
|
+
} catch (error) {
|
|
520
|
+
if (error.code === "EEXIST") {
|
|
521
|
+
await this.#atomicWrite(filePath, content);
|
|
522
|
+
} else {
|
|
523
|
+
throw error;
|
|
524
|
+
}
|
|
525
|
+
}
|
|
526
|
+
}
|
|
527
|
+
|
|
528
|
+
/**
|
|
529
|
+
* Applies edits to a file.
|
|
530
|
+
* @param {string} filePath - Path to the file
|
|
531
|
+
* @param {Array<{oldText: string, newText: string}>} edits - Array of edits
|
|
532
|
+
* @param {boolean} dryRun - If true, only preview changes
|
|
533
|
+
* @returns {Promise<string>} Diff of changes
|
|
534
|
+
*/
|
|
535
|
+
async applyFileEdits(filePath, edits, dryRun = false) {
|
|
536
|
+
const content = this.#normalizeLineEndings(
|
|
537
|
+
await fs.readFile(filePath, "utf-8")
|
|
538
|
+
);
|
|
539
|
+
|
|
540
|
+
let modifiedContent = content;
|
|
541
|
+
for (const edit of edits) {
|
|
542
|
+
const normalizedOld = this.#normalizeLineEndings(edit.oldText);
|
|
543
|
+
const normalizedNew = this.#normalizeLineEndings(edit.newText);
|
|
544
|
+
|
|
545
|
+
if (modifiedContent.includes(normalizedOld)) {
|
|
546
|
+
modifiedContent = modifiedContent.replace(normalizedOld, normalizedNew);
|
|
547
|
+
continue;
|
|
548
|
+
}
|
|
549
|
+
|
|
550
|
+
const oldLines = normalizedOld.split("\n");
|
|
551
|
+
const contentLines = modifiedContent.split("\n");
|
|
552
|
+
let matchFound = false;
|
|
553
|
+
|
|
554
|
+
for (let i = 0; i <= contentLines.length - oldLines.length; i++) {
|
|
555
|
+
const potentialMatch = contentLines.slice(i, i + oldLines.length);
|
|
556
|
+
|
|
557
|
+
const isMatch = oldLines.every((oldLine, j) => {
|
|
558
|
+
const contentLine = potentialMatch[j];
|
|
559
|
+
return oldLine.trim() === contentLine.trim();
|
|
560
|
+
});
|
|
561
|
+
|
|
562
|
+
if (isMatch) {
|
|
563
|
+
const originalIndent = contentLines[i].match(/^\s*/)?.[0] || "";
|
|
564
|
+
const newLines = normalizedNew.split("\n").map((line, j) => {
|
|
565
|
+
if (j === 0) return originalIndent + line.trimStart();
|
|
566
|
+
const oldIndent = oldLines[j]?.match(/^\s*/)?.[0] || "";
|
|
567
|
+
const newIndent = line.match(/^\s*/)?.[0] || "";
|
|
568
|
+
if (oldIndent && newIndent) {
|
|
569
|
+
const relativeIndent = newIndent.length - oldIndent.length;
|
|
570
|
+
return (
|
|
571
|
+
originalIndent +
|
|
572
|
+
" ".repeat(Math.max(0, relativeIndent)) +
|
|
573
|
+
line.trimStart()
|
|
574
|
+
);
|
|
575
|
+
}
|
|
576
|
+
return line;
|
|
577
|
+
});
|
|
578
|
+
|
|
579
|
+
contentLines.splice(i, oldLines.length, ...newLines);
|
|
580
|
+
modifiedContent = contentLines.join("\n");
|
|
581
|
+
matchFound = true;
|
|
582
|
+
break;
|
|
583
|
+
}
|
|
584
|
+
}
|
|
585
|
+
|
|
586
|
+
if (!matchFound) {
|
|
587
|
+
throw new Error(
|
|
588
|
+
`Could not find exact match for edit:\n${edit.oldText}`
|
|
589
|
+
);
|
|
590
|
+
}
|
|
591
|
+
}
|
|
592
|
+
|
|
593
|
+
const diffResult = this.#createUnifiedDiff(
|
|
594
|
+
content,
|
|
595
|
+
modifiedContent,
|
|
596
|
+
filePath
|
|
597
|
+
);
|
|
598
|
+
|
|
599
|
+
let numBackticks = 3;
|
|
600
|
+
while (diffResult.includes("`".repeat(numBackticks))) {
|
|
601
|
+
numBackticks++;
|
|
602
|
+
}
|
|
603
|
+
const formattedDiff = `${"`".repeat(numBackticks)}diff\n${diffResult}${"`".repeat(numBackticks)}\n\n`;
|
|
604
|
+
|
|
605
|
+
if (!dryRun) {
|
|
606
|
+
await this.#atomicWrite(filePath, modifiedContent);
|
|
607
|
+
}
|
|
608
|
+
|
|
609
|
+
return formattedDiff;
|
|
610
|
+
}
|
|
611
|
+
|
|
612
|
+
/**
|
|
613
|
+
* Gets the last N lines of a file.
|
|
614
|
+
* @param {string} filePath - Path to the file
|
|
615
|
+
* @param {number} numLines - Number of lines to return
|
|
616
|
+
* @returns {Promise<string>} Last N lines of the file
|
|
617
|
+
*/
|
|
618
|
+
async tailFile(filePath, numLines) {
|
|
619
|
+
return this.#withCollectorFallback(
|
|
620
|
+
filePath,
|
|
621
|
+
(content) => {
|
|
622
|
+
const lines = this.#normalizeLineEndings(content).split("\n");
|
|
623
|
+
return lines.slice(-numLines).join("\n");
|
|
624
|
+
},
|
|
625
|
+
() => this.#tailFileRaw(filePath, numLines)
|
|
626
|
+
);
|
|
627
|
+
}
|
|
628
|
+
|
|
629
|
+
/**
|
|
630
|
+
* Gets the first N lines of a file.
|
|
631
|
+
* @param {string} filePath - Path to the file
|
|
632
|
+
* @param {number} numLines - Number of lines to return
|
|
633
|
+
* @returns {Promise<string>} First N lines of the file
|
|
634
|
+
*/
|
|
635
|
+
async headFile(filePath, numLines) {
|
|
636
|
+
return this.#withCollectorFallback(
|
|
637
|
+
filePath,
|
|
638
|
+
(content) => {
|
|
639
|
+
const lines = this.#normalizeLineEndings(content).split("\n");
|
|
640
|
+
return lines.slice(0, numLines).join("\n");
|
|
641
|
+
},
|
|
642
|
+
() => this.#headFileRaw(filePath, numLines)
|
|
643
|
+
);
|
|
644
|
+
}
|
|
645
|
+
|
|
646
|
+
/**
|
|
647
|
+
* Searches for files matching a glob pattern.
|
|
648
|
+
* @param {string} rootPath - Root path to search from
|
|
649
|
+
* @param {string} pattern - Glob pattern to match
|
|
650
|
+
* @param {Object} options - Search options
|
|
651
|
+
* @param {string[]} options.excludePatterns - Patterns to exclude
|
|
652
|
+
* @returns {Promise<string[]>} Array of matching file paths
|
|
653
|
+
*/
|
|
654
|
+
async searchFilesWithGlob(rootPath, pattern, options = {}) {
|
|
655
|
+
const minimatch = require("minimatch");
|
|
656
|
+
const { excludePatterns = [] } = options;
|
|
657
|
+
const results = [];
|
|
658
|
+
const matchOptions = { dot: true, nocase: true };
|
|
659
|
+
|
|
660
|
+
const search = async (currentPath) => {
|
|
661
|
+
const entries = await fs.readdir(currentPath, { withFileTypes: true });
|
|
662
|
+
|
|
663
|
+
for (const entry of entries) {
|
|
664
|
+
const fullPath = path.join(currentPath, entry.name);
|
|
665
|
+
|
|
666
|
+
try {
|
|
667
|
+
await this.validatePath(fullPath);
|
|
668
|
+
|
|
669
|
+
const relativePath = path.relative(rootPath, fullPath);
|
|
670
|
+
const shouldExclude = excludePatterns.some(
|
|
671
|
+
(excludePattern) =>
|
|
672
|
+
minimatch(relativePath, excludePattern, matchOptions) ||
|
|
673
|
+
minimatch(entry.name, excludePattern, matchOptions)
|
|
674
|
+
);
|
|
675
|
+
|
|
676
|
+
if (shouldExclude) continue;
|
|
677
|
+
|
|
678
|
+
const matchesPath = minimatch(relativePath, pattern, matchOptions);
|
|
679
|
+
const matchesName = minimatch(entry.name, pattern, matchOptions);
|
|
680
|
+
|
|
681
|
+
if (matchesPath || matchesName) {
|
|
682
|
+
results.push(fullPath);
|
|
683
|
+
}
|
|
684
|
+
|
|
685
|
+
if (entry.isDirectory()) {
|
|
686
|
+
await search(fullPath);
|
|
687
|
+
}
|
|
688
|
+
} catch {
|
|
689
|
+
continue;
|
|
690
|
+
}
|
|
691
|
+
}
|
|
692
|
+
};
|
|
693
|
+
|
|
694
|
+
await search(rootPath);
|
|
695
|
+
return results;
|
|
696
|
+
}
|
|
697
|
+
|
|
698
|
+
/**
|
|
699
|
+
* Truncates content if it exceeds the model's context limit.
|
|
700
|
+
* @param {string} content - The content to potentially truncate
|
|
701
|
+
* @param {object} aibitat - The aibitat instance with model/provider info
|
|
702
|
+
* @param {string} [truncationMessage] - Optional custom message
|
|
703
|
+
* @returns {{content: string, wasTruncated: boolean}} The content and truncation status
|
|
704
|
+
*/
|
|
705
|
+
truncateContentForContext(content, aibitat, truncationMessage = null) {
|
|
706
|
+
const { TokenManager } = require("../../../../helpers/tiktoken");
|
|
707
|
+
const Provider = require("../../providers/ai-provider");
|
|
708
|
+
|
|
709
|
+
const contextLimit = Provider.contextLimit(aibitat.provider, aibitat.model);
|
|
710
|
+
const reserveForResponse = Math.floor(
|
|
711
|
+
contextLimit * FilesystemManager.CONTEXT_RESERVE_RATIO
|
|
712
|
+
);
|
|
713
|
+
const maxTokens = contextLimit - reserveForResponse;
|
|
714
|
+
|
|
715
|
+
const tokenManager = new TokenManager(aibitat.model);
|
|
716
|
+
const tokenCount = tokenManager.countFromString(content);
|
|
717
|
+
|
|
718
|
+
if (tokenCount <= maxTokens) {
|
|
719
|
+
return { content, wasTruncated: false };
|
|
720
|
+
}
|
|
721
|
+
|
|
722
|
+
const avgCharsPerToken = content.length / tokenCount;
|
|
723
|
+
let targetChars = Math.floor(maxTokens * avgCharsPerToken);
|
|
724
|
+
let truncated = content.slice(0, targetChars);
|
|
725
|
+
|
|
726
|
+
const lastNewline = truncated.lastIndexOf("\n");
|
|
727
|
+
if (lastNewline > targetChars * 0.8) {
|
|
728
|
+
truncated = truncated.slice(0, lastNewline);
|
|
729
|
+
}
|
|
730
|
+
|
|
731
|
+
const defaultMessage =
|
|
732
|
+
"[Content truncated - exceeds context limit. Consider reading smaller portions.]";
|
|
733
|
+
const message = truncationMessage || defaultMessage;
|
|
734
|
+
|
|
735
|
+
return {
|
|
736
|
+
content: truncated + "\n\n" + message,
|
|
737
|
+
wasTruncated: true,
|
|
738
|
+
};
|
|
739
|
+
}
|
|
740
|
+
|
|
741
|
+
/**
|
|
742
|
+
* Check if a file path points to an image file.
|
|
743
|
+
* @param {string} filePath - Path to the file
|
|
744
|
+
* @returns {boolean} True if the file is an image
|
|
745
|
+
*/
|
|
746
|
+
isImageFile(filePath) {
|
|
747
|
+
const ext = path.extname(filePath).toLowerCase();
|
|
748
|
+
return FilesystemManager.IMAGE_EXTENSIONS.includes(ext);
|
|
749
|
+
}
|
|
750
|
+
|
|
751
|
+
/**
|
|
752
|
+
* Get the MIME type for an image file.
|
|
753
|
+
* @param {string} filePath - Path to the file
|
|
754
|
+
* @returns {string|null} MIME type or null if not an image
|
|
755
|
+
*/
|
|
756
|
+
getImageMimeType(filePath) {
|
|
757
|
+
const ext = path.extname(filePath).toLowerCase();
|
|
758
|
+
return FilesystemManager.IMAGE_MIME_TYPES[ext] || null;
|
|
759
|
+
}
|
|
760
|
+
|
|
761
|
+
/**
|
|
762
|
+
* Read an image file and return it as an attachment object.
|
|
763
|
+
* @param {string} filePath - Validated absolute path to the image file
|
|
764
|
+
* @returns {Promise<{name: string, mime: string, contentString: string}|null>} Attachment object or null on error
|
|
765
|
+
*/
|
|
766
|
+
async readImageAsAttachment(filePath) {
|
|
767
|
+
try {
|
|
768
|
+
const mime = this.getImageMimeType(filePath);
|
|
769
|
+
if (!mime) return null;
|
|
770
|
+
|
|
771
|
+
const buffer = await fs.readFile(filePath);
|
|
772
|
+
const base64 = buffer.toString("base64");
|
|
773
|
+
const filename = path.basename(filePath);
|
|
774
|
+
|
|
775
|
+
return {
|
|
776
|
+
name: filename,
|
|
777
|
+
mime,
|
|
778
|
+
contentString: `data:${mime};base64,${base64}`,
|
|
779
|
+
};
|
|
780
|
+
} catch (error) {
|
|
781
|
+
console.error(`Error reading image file ${filePath}:`, error.message);
|
|
782
|
+
return null;
|
|
783
|
+
}
|
|
784
|
+
}
|
|
785
|
+
}
|
|
786
|
+
|
|
787
|
+
module.exports = new FilesystemManager();
|