vectra 0.12.3 → 0.14.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (375) hide show
  1. package/README.md +92 -100
  2. package/lib/BrowserWebFetcher.d.ts +75 -0
  3. package/lib/BrowserWebFetcher.d.ts.map +1 -0
  4. package/lib/BrowserWebFetcher.js +290 -0
  5. package/lib/BrowserWebFetcher.js.map +1 -0
  6. package/lib/FileFetcher.d.ts.map +1 -1
  7. package/lib/FileFetcher.js +25 -15
  8. package/lib/FileFetcher.js.map +1 -1
  9. package/lib/FileFetcher.spec.d.ts +2 -0
  10. package/lib/FileFetcher.spec.d.ts.map +1 -0
  11. package/lib/FileFetcher.spec.js +244 -0
  12. package/lib/FileFetcher.spec.js.map +1 -0
  13. package/lib/FolderWatcher.d.ts +91 -0
  14. package/lib/FolderWatcher.d.ts.map +1 -0
  15. package/lib/FolderWatcher.js +304 -0
  16. package/lib/FolderWatcher.js.map +1 -0
  17. package/lib/FolderWatcher.spec.d.ts +2 -0
  18. package/lib/FolderWatcher.spec.d.ts.map +1 -0
  19. package/lib/FolderWatcher.spec.js +308 -0
  20. package/lib/FolderWatcher.spec.js.map +1 -0
  21. package/lib/GPT3Tokenizer.spec.d.ts +2 -0
  22. package/lib/GPT3Tokenizer.spec.d.ts.map +1 -0
  23. package/lib/GPT3Tokenizer.spec.js +45 -0
  24. package/lib/GPT3Tokenizer.spec.js.map +1 -0
  25. package/lib/ItemSelector.d.ts.map +1 -1
  26. package/lib/ItemSelector.js +19 -8
  27. package/lib/ItemSelector.js.map +1 -1
  28. package/lib/ItemSelector.spec.d.ts +2 -0
  29. package/lib/ItemSelector.spec.d.ts.map +1 -0
  30. package/lib/ItemSelector.spec.js +204 -0
  31. package/lib/ItemSelector.spec.js.map +1 -0
  32. package/lib/LocalDocument.d.ts +1 -1
  33. package/lib/LocalDocument.d.ts.map +1 -1
  34. package/lib/LocalDocument.js +5 -45
  35. package/lib/LocalDocument.js.map +1 -1
  36. package/lib/LocalDocument.spec.d.ts +2 -0
  37. package/lib/LocalDocument.spec.d.ts.map +1 -0
  38. package/lib/LocalDocument.spec.js +214 -0
  39. package/lib/LocalDocument.spec.js.map +1 -0
  40. package/lib/LocalDocumentIndex.d.ts +20 -0
  41. package/lib/LocalDocumentIndex.d.ts.map +1 -1
  42. package/lib/LocalDocumentIndex.js +16 -52
  43. package/lib/LocalDocumentIndex.js.map +1 -1
  44. package/lib/LocalDocumentIndex.spec.d.ts +2 -0
  45. package/lib/LocalDocumentIndex.spec.d.ts.map +1 -0
  46. package/lib/LocalDocumentIndex.spec.js +494 -0
  47. package/lib/LocalDocumentIndex.spec.js.map +1 -0
  48. package/lib/LocalDocumentResult.d.ts +32 -11
  49. package/lib/LocalDocumentResult.d.ts.map +1 -1
  50. package/lib/LocalDocumentResult.js +305 -257
  51. package/lib/LocalDocumentResult.js.map +1 -1
  52. package/lib/LocalDocumentResult.spec.d.ts +2 -0
  53. package/lib/LocalDocumentResult.spec.d.ts.map +1 -0
  54. package/lib/LocalDocumentResult.spec.js +373 -0
  55. package/lib/LocalDocumentResult.spec.js.map +1 -0
  56. package/lib/LocalEmbeddings.d.ts +59 -0
  57. package/lib/LocalEmbeddings.d.ts.map +1 -0
  58. package/lib/LocalEmbeddings.js +101 -0
  59. package/lib/LocalEmbeddings.js.map +1 -0
  60. package/lib/LocalEmbeddings.spec.d.ts +2 -0
  61. package/lib/LocalEmbeddings.spec.d.ts.map +1 -0
  62. package/lib/LocalEmbeddings.spec.js +155 -0
  63. package/lib/LocalEmbeddings.spec.js.map +1 -0
  64. package/lib/LocalIndex.d.ts +27 -18
  65. package/lib/LocalIndex.d.ts.map +1 -1
  66. package/lib/LocalIndex.js +109 -105
  67. package/lib/LocalIndex.js.map +1 -1
  68. package/lib/LocalIndex.spec.js +434 -43
  69. package/lib/LocalIndex.spec.js.map +1 -1
  70. package/lib/OpenAIEmbeddings.d.ts +4 -6
  71. package/lib/OpenAIEmbeddings.d.ts.map +1 -1
  72. package/lib/OpenAIEmbeddings.js +16 -24
  73. package/lib/OpenAIEmbeddings.js.map +1 -1
  74. package/lib/OpenAIEmbeddings.spec.d.ts +2 -0
  75. package/lib/OpenAIEmbeddings.spec.d.ts.map +1 -0
  76. package/lib/OpenAIEmbeddings.spec.js +298 -0
  77. package/lib/OpenAIEmbeddings.spec.js.map +1 -0
  78. package/lib/TextSplitter.d.ts +2 -0
  79. package/lib/TextSplitter.d.ts.map +1 -1
  80. package/lib/TextSplitter.js +154 -111
  81. package/lib/TextSplitter.js.map +1 -1
  82. package/lib/TextSplitter.spec.js +289 -61
  83. package/lib/TextSplitter.spec.js.map +1 -1
  84. package/lib/TransformersEmbeddings.d.ts +121 -0
  85. package/lib/TransformersEmbeddings.d.ts.map +1 -0
  86. package/lib/TransformersEmbeddings.js +176 -0
  87. package/lib/TransformersEmbeddings.js.map +1 -0
  88. package/lib/TransformersEmbeddings.spec.d.ts +2 -0
  89. package/lib/TransformersEmbeddings.spec.d.ts.map +1 -0
  90. package/lib/TransformersEmbeddings.spec.js +198 -0
  91. package/lib/TransformersEmbeddings.spec.js.map +1 -0
  92. package/lib/TransformersTokenizer.d.ts +33 -0
  93. package/lib/TransformersTokenizer.d.ts.map +1 -0
  94. package/lib/TransformersTokenizer.js +44 -0
  95. package/lib/TransformersTokenizer.js.map +1 -0
  96. package/lib/TransformersTokenizer.spec.d.ts +2 -0
  97. package/lib/TransformersTokenizer.spec.d.ts.map +1 -0
  98. package/lib/TransformersTokenizer.spec.js +112 -0
  99. package/lib/TransformersTokenizer.spec.js.map +1 -0
  100. package/lib/WebFetcher.d.ts +1 -2
  101. package/lib/WebFetcher.d.ts.map +1 -1
  102. package/lib/WebFetcher.js +58 -54
  103. package/lib/WebFetcher.js.map +1 -1
  104. package/lib/WebFetcher.spec.d.ts +2 -0
  105. package/lib/WebFetcher.spec.d.ts.map +1 -0
  106. package/lib/WebFetcher.spec.js +263 -0
  107. package/lib/WebFetcher.spec.js.map +1 -0
  108. package/lib/browser.d.ts +30 -0
  109. package/lib/browser.d.ts.map +1 -0
  110. package/lib/browser.js +52 -0
  111. package/lib/browser.js.map +1 -0
  112. package/lib/codecs/IndexCodec.d.ts +37 -0
  113. package/lib/codecs/IndexCodec.d.ts.map +1 -0
  114. package/lib/codecs/IndexCodec.js +3 -0
  115. package/lib/codecs/IndexCodec.js.map +1 -0
  116. package/lib/codecs/JsonCodec.d.ts +19 -0
  117. package/lib/codecs/JsonCodec.d.ts.map +1 -0
  118. package/lib/codecs/JsonCodec.js +35 -0
  119. package/lib/codecs/JsonCodec.js.map +1 -0
  120. package/lib/codecs/JsonCodec.spec.d.ts +2 -0
  121. package/lib/codecs/JsonCodec.spec.d.ts.map +1 -0
  122. package/lib/codecs/JsonCodec.spec.js +66 -0
  123. package/lib/codecs/JsonCodec.spec.js.map +1 -0
  124. package/lib/codecs/LocalIndex.protobuf.spec.d.ts +2 -0
  125. package/lib/codecs/LocalIndex.protobuf.spec.d.ts.map +1 -0
  126. package/lib/codecs/LocalIndex.protobuf.spec.js +108 -0
  127. package/lib/codecs/LocalIndex.protobuf.spec.js.map +1 -0
  128. package/lib/codecs/ProtobufCodec.d.ts +20 -0
  129. package/lib/codecs/ProtobufCodec.d.ts.map +1 -0
  130. package/lib/codecs/ProtobufCodec.js +225 -0
  131. package/lib/codecs/ProtobufCodec.js.map +1 -0
  132. package/lib/codecs/ProtobufCodec.spec.d.ts +2 -0
  133. package/lib/codecs/ProtobufCodec.spec.d.ts.map +1 -0
  134. package/lib/codecs/ProtobufCodec.spec.js +155 -0
  135. package/lib/codecs/ProtobufCodec.spec.js.map +1 -0
  136. package/lib/codecs/index.d.ts +5 -0
  137. package/lib/codecs/index.d.ts.map +1 -0
  138. package/lib/codecs/index.js +21 -0
  139. package/lib/codecs/index.js.map +1 -0
  140. package/lib/codecs/migrateIndex.d.ts +24 -0
  141. package/lib/codecs/migrateIndex.d.ts.map +1 -0
  142. package/lib/codecs/migrateIndex.js +119 -0
  143. package/lib/codecs/migrateIndex.js.map +1 -0
  144. package/lib/codecs/migrateIndex.spec.d.ts +2 -0
  145. package/lib/codecs/migrateIndex.spec.d.ts.map +1 -0
  146. package/lib/codecs/migrateIndex.spec.js +151 -0
  147. package/lib/codecs/migrateIndex.spec.js.map +1 -0
  148. package/lib/codecs/schemas/index.proto +34 -0
  149. package/lib/index.d.ts +9 -1
  150. package/lib/index.d.ts.map +1 -1
  151. package/lib/index.js +9 -1
  152. package/lib/index.js.map +1 -1
  153. package/lib/internals/Colorize.d.ts.map +1 -1
  154. package/lib/internals/Colorize.js +20 -15
  155. package/lib/internals/Colorize.js.map +1 -1
  156. package/lib/server/IndexManager.d.ts +78 -0
  157. package/lib/server/IndexManager.d.ts.map +1 -0
  158. package/lib/server/IndexManager.js +259 -0
  159. package/lib/server/IndexManager.js.map +1 -0
  160. package/lib/server/VectraServer.d.ts +40 -0
  161. package/lib/server/VectraServer.d.ts.map +1 -0
  162. package/lib/server/VectraServer.js +151 -0
  163. package/lib/server/VectraServer.js.map +1 -0
  164. package/lib/server/VectraServer.spec.d.ts +2 -0
  165. package/lib/server/VectraServer.spec.d.ts.map +1 -0
  166. package/lib/server/VectraServer.spec.js +322 -0
  167. package/lib/server/VectraServer.spec.js.map +1 -0
  168. package/lib/server/handlers/documentHandlers.d.ts +15 -0
  169. package/lib/server/handlers/documentHandlers.d.ts.map +1 -0
  170. package/lib/server/handlers/documentHandlers.js +95 -0
  171. package/lib/server/handlers/documentHandlers.js.map +1 -0
  172. package/lib/server/handlers/helpers.d.ts +23 -0
  173. package/lib/server/handlers/helpers.d.ts.map +1 -0
  174. package/lib/server/handlers/helpers.js +138 -0
  175. package/lib/server/handlers/helpers.js.map +1 -0
  176. package/lib/server/handlers/index.d.ts +8 -0
  177. package/lib/server/handlers/index.d.ts.map +1 -0
  178. package/lib/server/handlers/index.js +22 -0
  179. package/lib/server/handlers/index.js.map +1 -0
  180. package/lib/server/handlers/indexHandlers.d.ts +14 -0
  181. package/lib/server/handlers/indexHandlers.d.ts.map +1 -0
  182. package/lib/server/handlers/indexHandlers.js +85 -0
  183. package/lib/server/handlers/indexHandlers.js.map +1 -0
  184. package/lib/server/handlers/itemHandlers.d.ts +34 -0
  185. package/lib/server/handlers/itemHandlers.d.ts.map +1 -0
  186. package/lib/server/handlers/itemHandlers.js +166 -0
  187. package/lib/server/handlers/itemHandlers.js.map +1 -0
  188. package/lib/server/handlers/lifecycleHandlers.d.ts +11 -0
  189. package/lib/server/handlers/lifecycleHandlers.d.ts.map +1 -0
  190. package/lib/server/handlers/lifecycleHandlers.js +31 -0
  191. package/lib/server/handlers/lifecycleHandlers.js.map +1 -0
  192. package/lib/server/handlers/queryHandlers.d.ts +27 -0
  193. package/lib/server/handlers/queryHandlers.d.ts.map +1 -0
  194. package/lib/server/handlers/queryHandlers.js +135 -0
  195. package/lib/server/handlers/queryHandlers.js.map +1 -0
  196. package/lib/server/handlers/statsHandlers.d.ts +17 -0
  197. package/lib/server/handlers/statsHandlers.d.ts.map +1 -0
  198. package/lib/server/handlers/statsHandlers.js +81 -0
  199. package/lib/server/handlers/statsHandlers.js.map +1 -0
  200. package/lib/server/index.d.ts +4 -0
  201. package/lib/server/index.d.ts.map +1 -0
  202. package/lib/server/index.js +23 -0
  203. package/lib/server/index.js.map +1 -0
  204. package/lib/storage/FileStorage.d.ts +92 -0
  205. package/lib/storage/FileStorage.d.ts.map +1 -0
  206. package/lib/storage/FileStorage.js +3 -0
  207. package/lib/storage/FileStorage.js.map +1 -0
  208. package/lib/storage/FileStorageUtilities.d.ts +36 -0
  209. package/lib/storage/FileStorageUtilities.d.ts.map +1 -0
  210. package/lib/storage/FileStorageUtilities.js +91 -0
  211. package/lib/storage/FileStorageUtilities.js.map +1 -0
  212. package/lib/storage/FileStorageUtilities.spec.d.ts +2 -0
  213. package/lib/storage/FileStorageUtilities.spec.d.ts.map +1 -0
  214. package/lib/storage/FileStorageUtilities.spec.js +98 -0
  215. package/lib/storage/FileStorageUtilities.spec.js.map +1 -0
  216. package/lib/storage/FileType.d.ts +29 -0
  217. package/lib/storage/FileType.d.ts.map +1 -0
  218. package/lib/storage/FileType.js +38 -0
  219. package/lib/storage/FileType.js.map +1 -0
  220. package/lib/storage/IndexedDBStorage.d.ts +47 -0
  221. package/lib/storage/IndexedDBStorage.d.ts.map +1 -0
  222. package/lib/storage/IndexedDBStorage.js +347 -0
  223. package/lib/storage/IndexedDBStorage.js.map +1 -0
  224. package/lib/storage/LocalFileStorage.browser.d.ts +19 -0
  225. package/lib/storage/LocalFileStorage.browser.d.ts.map +1 -0
  226. package/lib/storage/LocalFileStorage.browser.js +43 -0
  227. package/lib/storage/LocalFileStorage.browser.js.map +1 -0
  228. package/lib/storage/LocalFileStorage.d.ts +23 -0
  229. package/lib/storage/LocalFileStorage.d.ts.map +1 -0
  230. package/lib/storage/LocalFileStorage.js +152 -0
  231. package/lib/storage/LocalFileStorage.js.map +1 -0
  232. package/lib/storage/LocalFileStorage.spec.d.ts +2 -0
  233. package/lib/storage/LocalFileStorage.spec.d.ts.map +1 -0
  234. package/lib/storage/LocalFileStorage.spec.js +249 -0
  235. package/lib/storage/LocalFileStorage.spec.js.map +1 -0
  236. package/lib/storage/VirtualFileStorage.d.ts +18 -0
  237. package/lib/storage/VirtualFileStorage.d.ts.map +1 -0
  238. package/lib/storage/VirtualFileStorage.js +178 -0
  239. package/lib/storage/VirtualFileStorage.js.map +1 -0
  240. package/lib/storage/VirtualFileStorage.spec.d.ts +2 -0
  241. package/lib/storage/VirtualFileStorage.spec.d.ts.map +1 -0
  242. package/lib/storage/VirtualFileStorage.spec.js +302 -0
  243. package/lib/storage/VirtualFileStorage.spec.js.map +1 -0
  244. package/lib/storage/index.d.ts +6 -0
  245. package/lib/storage/index.d.ts.map +1 -0
  246. package/lib/storage/index.js +22 -0
  247. package/lib/storage/index.js.map +1 -0
  248. package/lib/templates/templates/csharp/README.md +48 -0
  249. package/lib/templates/templates/csharp/VectraClient.cs +234 -0
  250. package/lib/templates/templates/go/README.md +71 -0
  251. package/lib/templates/templates/go/vectra_client.go +322 -0
  252. package/lib/templates/templates/java/README.md +81 -0
  253. package/lib/templates/templates/java/VectraClient.java +232 -0
  254. package/lib/templates/templates/python/README.md +37 -0
  255. package/lib/templates/templates/python/vectra_client.py +279 -0
  256. package/lib/templates/templates/rust/Cargo.toml +14 -0
  257. package/lib/templates/templates/rust/README.md +39 -0
  258. package/lib/templates/templates/rust/build.rs +4 -0
  259. package/lib/templates/templates/rust/lib.rs +284 -0
  260. package/lib/templates/templates/typescript/README.md +96 -0
  261. package/lib/templates/templates/typescript/VectraClient.ts +374 -0
  262. package/lib/templates/typescript/VectraClient.d.ts +114 -0
  263. package/lib/templates/typescript/VectraClient.d.ts.map +1 -0
  264. package/lib/templates/typescript/VectraClient.js +328 -0
  265. package/lib/templates/typescript/VectraClient.js.map +1 -0
  266. package/lib/types.d.ts +7 -0
  267. package/lib/types.d.ts.map +1 -1
  268. package/lib/utils/index.d.ts +2 -0
  269. package/lib/utils/index.d.ts.map +1 -0
  270. package/lib/utils/index.js +18 -0
  271. package/lib/utils/index.js.map +1 -0
  272. package/lib/utils/pathUtils.d.ts +40 -0
  273. package/lib/utils/pathUtils.d.ts.map +1 -0
  274. package/lib/utils/pathUtils.js +98 -0
  275. package/lib/utils/pathUtils.js.map +1 -0
  276. package/lib/vectra-cli.d.ts.map +1 -1
  277. package/lib/vectra-cli.generate.spec.d.ts +2 -0
  278. package/lib/vectra-cli.generate.spec.d.ts.map +1 -0
  279. package/lib/vectra-cli.generate.spec.js +112 -0
  280. package/lib/vectra-cli.generate.spec.js.map +1 -0
  281. package/lib/vectra-cli.js +446 -9
  282. package/lib/vectra-cli.js.map +1 -1
  283. package/lib/vectra-cli.spec.d.ts +1 -0
  284. package/lib/vectra-cli.spec.d.ts.map +1 -0
  285. package/lib/vectra-cli.spec.js +2 -0
  286. package/lib/vectra-cli.spec.js.map +1 -0
  287. package/package.json +89 -16
  288. package/proto/vectra_service.proto +276 -0
  289. package/src/BrowserWebFetcher.ts +345 -0
  290. package/src/FileFetcher.spec.ts +234 -0
  291. package/src/FileFetcher.ts +37 -25
  292. package/src/FolderWatcher.spec.ts +288 -0
  293. package/src/FolderWatcher.ts +304 -0
  294. package/src/GPT3Tokenizer.spec.ts +50 -0
  295. package/src/ItemSelector.spec.ts +252 -0
  296. package/src/ItemSelector.ts +163 -150
  297. package/src/LocalDocument.spec.ts +211 -0
  298. package/src/LocalDocument.ts +88 -94
  299. package/src/LocalDocumentIndex.spec.ts +481 -0
  300. package/src/LocalDocumentIndex.ts +39 -40
  301. package/src/LocalDocumentResult.spec.ts +373 -0
  302. package/src/LocalDocumentResult.ts +489 -319
  303. package/src/LocalEmbeddings.spec.ts +138 -0
  304. package/src/LocalEmbeddings.ts +120 -0
  305. package/src/LocalIndex.spec.ts +808 -323
  306. package/src/LocalIndex.ts +479 -430
  307. package/src/OpenAIEmbeddings.spec.ts +354 -0
  308. package/src/OpenAIEmbeddings.ts +26 -27
  309. package/src/TextSplitter.spec.ts +320 -65
  310. package/src/TextSplitter.ts +172 -115
  311. package/src/TransformersEmbeddings.spec.ts +188 -0
  312. package/src/TransformersEmbeddings.ts +232 -0
  313. package/src/TransformersTokenizer.spec.ts +143 -0
  314. package/src/TransformersTokenizer.ts +45 -0
  315. package/src/WebFetcher.spec.ts +288 -0
  316. package/src/WebFetcher.ts +184 -186
  317. package/src/browser.ts +69 -0
  318. package/src/codecs/IndexCodec.ts +40 -0
  319. package/src/codecs/JsonCodec.spec.ts +70 -0
  320. package/src/codecs/JsonCodec.ts +37 -0
  321. package/src/codecs/LocalIndex.protobuf.spec.ts +115 -0
  322. package/src/codecs/ProtobufCodec.spec.ts +166 -0
  323. package/src/codecs/ProtobufCodec.ts +193 -0
  324. package/src/codecs/index.ts +4 -0
  325. package/src/codecs/migrateIndex.spec.ts +176 -0
  326. package/src/codecs/migrateIndex.ts +125 -0
  327. package/src/codecs/schemas/index.proto +34 -0
  328. package/src/index.ts +9 -1
  329. package/src/internals/Colorize.ts +19 -16
  330. package/src/server/IndexManager.ts +243 -0
  331. package/src/server/VectraServer.spec.ts +303 -0
  332. package/src/server/VectraServer.ts +156 -0
  333. package/src/server/handlers/documentHandlers.ts +59 -0
  334. package/src/server/handlers/helpers.ts +93 -0
  335. package/src/server/handlers/index.ts +7 -0
  336. package/src/server/handlers/indexHandlers.ts +44 -0
  337. package/src/server/handlers/itemHandlers.ts +140 -0
  338. package/src/server/handlers/lifecycleHandlers.ts +26 -0
  339. package/src/server/handlers/queryHandlers.ts +96 -0
  340. package/src/server/handlers/statsHandlers.ts +38 -0
  341. package/src/server/index.ts +3 -0
  342. package/src/storage/FileStorage.ts +105 -0
  343. package/src/storage/FileStorageUtilities.spec.ts +106 -0
  344. package/src/storage/FileStorageUtilities.ts +77 -0
  345. package/src/storage/FileType.ts +61 -0
  346. package/src/storage/IndexedDBStorage.ts +365 -0
  347. package/src/storage/LocalFileStorage.browser.ts +52 -0
  348. package/src/storage/LocalFileStorage.spec.ts +292 -0
  349. package/src/storage/LocalFileStorage.ts +98 -0
  350. package/src/storage/VirtualFileStorage.spec.ts +307 -0
  351. package/src/storage/VirtualFileStorage.ts +169 -0
  352. package/src/storage/index.ts +5 -0
  353. package/src/templates/csharp/README.md +48 -0
  354. package/src/templates/csharp/VectraClient.cs +234 -0
  355. package/src/templates/go/README.md +71 -0
  356. package/src/templates/go/vectra_client.go +322 -0
  357. package/src/templates/java/README.md +81 -0
  358. package/src/templates/java/VectraClient.java +232 -0
  359. package/src/templates/python/README.md +37 -0
  360. package/src/templates/python/vectra_client.py +279 -0
  361. package/src/templates/rust/Cargo.toml +14 -0
  362. package/src/templates/rust/README.md +39 -0
  363. package/src/templates/rust/build.rs +4 -0
  364. package/src/templates/rust/lib.rs +284 -0
  365. package/src/templates/typescript/README.md +96 -0
  366. package/src/templates/typescript/VectraClient.ts +374 -0
  367. package/src/types.ts +131 -123
  368. package/src/utils/index.ts +1 -0
  369. package/src/utils/pathUtils.ts +106 -0
  370. package/src/vectra-cli.generate.spec.ts +72 -0
  371. package/src/vectra-cli.spec.ts +0 -0
  372. package/src/vectra-cli.ts +687 -246
  373. package/README.draft.md +0 -499
  374. package/README.draft.outline.md +0 -160
  375. package/README.research.md +0 -2159
package/src/WebFetcher.spec.ts ADDED
@@ -0,0 +1,288 @@
1
+ import assert from "node:assert";
2
+ import { describe, it, beforeEach, afterEach } from "mocha";
3
+ import sinon from "sinon";
4
+ import { WebFetcher } from "./WebFetcher";
5
+
6
+ describe("WebFetcher", () => {
7
+ let sandbox: sinon.SinonSandbox;
8
+ let fetchStub: sinon.SinonStub;
9
+
10
+ let lastUrl: string | undefined;
11
+ let lastInit: RequestInit | undefined;
12
+
13
+ function makeFetchResponse(status: number, data: string, contentType: string): Response {
14
+ return {
15
+ status,
16
+ statusText: status >= 400 ? `HTTP ${status}` : 'OK',
17
+ ok: status >= 200 && status < 300,
18
+ headers: new Headers({ 'content-type': contentType }),
19
+ text: async () => data,
20
+ json: async () => JSON.parse(data),
21
+ } as Response;
22
+ }
23
+
24
+ beforeEach(() => {
25
+ sandbox = sinon.createSandbox();
26
+ fetchStub = sandbox.stub(globalThis, 'fetch').callsFake(async (input: any, init?: any) => {
27
+ lastUrl = typeof input === 'string' ? input : input.url;
28
+ lastInit = init;
29
+ return makeFetchResponse(200, 'ok', 'text/plain');
30
+ });
31
+ lastUrl = undefined;
32
+ lastInit = undefined;
33
+ });
34
+
35
+ afterEach(() => {
36
+ sandbox.restore();
37
+ });
38
+
39
+ it("constructor uses defaults and merges config", () => {
40
+ const fetcherDefault = new WebFetcher();
41
+ assert.strictEqual(fetcherDefault["_config"].htmlToMarkdown, true);
42
+ assert.strictEqual(fetcherDefault["_config"].summarizeHtml, false);
43
+
44
+ const custom = {
45
+ htmlToMarkdown: false,
46
+ summarizeHtml: true,
47
+ headers: { Accept: "custom/type", "User-Agent": "custom-agent" },
48
+ requestConfig: { signal: AbortSignal.timeout(123) },
49
+ };
50
+ const fetcher = new WebFetcher(custom);
51
+ assert.strictEqual(fetcher["_config"].htmlToMarkdown, false);
52
+ assert.strictEqual(fetcher["_config"].summarizeHtml, true);
53
+ assert.deepStrictEqual(fetcher["_config"].headers, {
54
+ Accept: "custom/type",
55
+ "User-Agent": "custom-agent",
56
+ });
57
+ assert.ok(fetcher["_config"].requestConfig);
58
+ });
59
+
60
+ it("throws on HTTP error (>=400)", async () => {
61
+ fetchStub.resolves(makeFetchResponse(404, '<html></html>', 'text/html'));
62
+ const fetcher = new WebFetcher();
63
+ await assert.rejects(
64
+ () => fetcher.fetch("https://example.com/404", async () => true),
65
+ /Site returned an HTTP status of 404/
66
+ );
67
+ });
68
+
69
+ it("throws on invalid content-type", async () => {
70
+ fetchStub.resolves(makeFetchResponse(200, '…', 'image/png'));
71
+ const fetcher = new WebFetcher();
72
+ await assert.rejects(
73
+ () => fetcher.fetch("https://example.com/img", async () => true),
74
+ /Site returned an invalid content type of image\/png/
75
+ );
76
+ });
77
+
78
+ it("handles text/html with htmlToMarkdown=true with tables, alignments, and link rewriting", async () => {
79
+ const html = `
80
+ <html><body>
81
+ <script>evil()</script>
82
+ <a href="/rel">relative</a>
83
+ <a href="//cdn.example.com/lib.js">proto-rel</a>
84
+ <table>
85
+ <thead>
86
+ <tr>
87
+ <th align="left">H1</th>
88
+ <th align="right">H2</th>
89
+ <th align="center">H3</th>
90
+ </tr>
91
+ </thead>
92
+ <tbody>
93
+ <tr>
94
+ <td>Cell&nbsp;1</td>
95
+ <td>Cell\t2</td>
96
+ <td>Cell\n3</td>
97
+ </tr>
98
+ </tbody>
99
+ </table>
100
+ <table>
101
+ <tbody>
102
+ <tr><th>A</th><th>B</th></tr>
103
+ <tr><td>a</td><td>b</td></tr>
104
+ </tbody>
105
+ </table>
106
+ </body></html>
107
+ `;
108
+ fetchStub.resolves(makeFetchResponse(200, html, 'text/html'));
109
+
110
+ const fetcher = new WebFetcher({ htmlToMarkdown: true });
111
+ const calls: any[] = [];
112
+ const onDocument = async (uri: string, text: string, docType?: string) => {
113
+ calls.push([uri, text, docType]);
114
+ return true;
115
+ };
116
+
117
+ const ok = await fetcher.fetch("https://example.com/page", onDocument);
118
+ assert.strictEqual(ok, true);
119
+ assert.strictEqual(calls.length, 1);
120
+
121
+ const [uri, md, docType] = calls[0];
122
+ assert.strictEqual(uri, "https://example.com/page");
123
+ assert.strictEqual(docType, "md");
124
+
125
+ assert.ok(!String(md).includes("<script>"));
126
+ assert.ok(String(md).includes("https://example.com/rel"));
127
+ assert.ok(String(md).includes("https://cdn.example.com/lib.js"));
128
+ assert.ok(String(md).includes("| H1 | H2 | H3 |"));
129
+ assert.ok(String(md).includes("| :-- | --: | :-: |"));
130
+ assert.ok(String(md).includes("| Cell 1 | Cell 2 | Cell 3 |"));
131
+ assert.ok(String(md).includes("| A | B |"));
132
+ });
133
+
134
+ it("handles text/html with htmlToMarkdown=false (passes raw html, docType 'html')", async () => {
135
+ const html = "<html><body><p>content</p></body></html>";
136
+ fetchStub.resolves(makeFetchResponse(200, html, 'text/html'));
137
+
138
+ const fetcher = new WebFetcher({ htmlToMarkdown: false });
139
+ const calls: any[] = [];
140
+ const onDocument = async (uri: string, text: string, docType?: string) => {
141
+ calls.push([uri, text, docType]);
142
+ return true;
143
+ };
144
+
145
+ const ok = await fetcher.fetch("https://example.com/raw", onDocument);
146
+ assert.strictEqual(ok, true);
147
+ assert.strictEqual(calls.length, 1);
148
+
149
+ const [_, text, docType] = calls[0];
150
+ assert.strictEqual(docType, "html");
151
+ assert.strictEqual(text, html);
152
+ });
153
+
154
+ it("handles application/json; charset=… (docType 'json')", async () => {
155
+ const json = '{"a":1}';
156
+ fetchStub.resolves(makeFetchResponse(200, json, 'application/json; charset=utf-8'));
157
+ const fetcher = new WebFetcher();
158
+ const calls: any[] = [];
159
+ const onDocument = async (...args: any[]) => {
160
+ calls.push(args);
161
+ return true;
162
+ };
163
+ const ok = await fetcher.fetch("https://example.com/data", onDocument);
164
+ assert.strictEqual(ok, true);
165
+ assert.strictEqual(calls[0][2], "json");
166
+ assert.strictEqual(calls[0][1], json);
167
+ });
168
+
169
+ it("handles application/xml (docType 'xml')", async () => {
170
+ const xml = "<root/>";
171
+ fetchStub.resolves(makeFetchResponse(200, xml, 'application/xml'));
172
+ const fetcher = new WebFetcher();
173
+ const calls: any[] = [];
174
+ const onDocument = async (...args: any[]) => {
175
+ calls.push(args);
176
+ return true;
177
+ };
178
+ const ok = await fetcher.fetch("https://example.com/xml", onDocument);
179
+ assert.strictEqual(ok, true);
180
+ assert.strictEqual(calls[0][2], "xml");
181
+ assert.strictEqual(calls[0][1], xml);
182
+ });
183
+
184
+ it("handles application/javascript (docType 'javascript')", async () => {
185
+ const js = "console.log('hi');";
186
+ fetchStub.resolves(makeFetchResponse(200, js, 'application/javascript'));
187
+ const fetcher = new WebFetcher();
188
+ const calls: any[] = [];
189
+ const onDocument = async (...args: any[]) => {
190
+ calls.push(args);
191
+ return true;
192
+ };
193
+ const ok = await fetcher.fetch("https://example.com/app.js", onDocument);
194
+ assert.strictEqual(ok, true);
195
+ assert.strictEqual(calls[0][2], "javascript");
196
+ assert.strictEqual(calls[0][1], js);
197
+ });
198
+
199
+ it("handles text/plain (docType undefined)", async () => {
200
+ const text = "plain";
201
+ fetchStub.resolves(makeFetchResponse(200, text, 'text/plain'));
202
+ const fetcher = new WebFetcher();
203
+ const calls: any[] = [];
204
+ const onDocument = async (...args: any[]) => {
205
+ calls.push(args);
206
+ return true;
207
+ };
208
+ const ok = await fetcher.fetch("https://example.com/txt", onDocument);
209
+ assert.strictEqual(ok, true);
210
+ assert.strictEqual(calls[0][2], undefined);
211
+ assert.strictEqual(calls[0][1], text);
212
+ });
213
+
214
+ it("sets Host and Alt-Used to request hostname; merges and does not mutate caller headers", async () => {
215
+ let capturedUrl: string | undefined;
216
+ let capturedInit: RequestInit | undefined;
217
+ fetchStub.callsFake(async (url: string, init?: RequestInit) => {
218
+ capturedUrl = url;
219
+ capturedInit = init;
220
+ return makeFetchResponse(200, 'ok', 'text/plain');
221
+ });
222
+
223
+ const userHeaders: Record<string, string> = {
224
+ Accept: "custom/type",
225
+ "User-Agent": "custom-agent",
226
+ };
227
+ const fetcher = new WebFetcher({ headers: userHeaders });
228
+
229
+ const onDocument = async () => true;
230
+ const url = "https://host.example.com/path?q=1";
231
+ const ok = await fetcher.fetch(url, onDocument);
232
+ assert.strictEqual(ok, true);
233
+
234
+ assert.strictEqual(capturedUrl, url);
235
+ assert.ok(capturedInit);
236
+ assert.ok(capturedInit!.headers);
237
+
238
+ const headers = capturedInit!.headers as Record<string, string>;
239
+ assert.strictEqual(headers.Host, "host.example.com");
240
+ assert.strictEqual(headers["Alt-Used"], "host.example.com");
241
+
242
+ assert.strictEqual(headers.Accept, "custom/type");
243
+ assert.strictEqual(headers["User-Agent"], "custom-agent");
244
+
245
+ assert.strictEqual((userHeaders as any).Host, undefined);
246
+ assert.strictEqual((userHeaders as any)["Alt-Used"], undefined);
247
+ });
248
+
249
+ it("merges requestConfig options into fetch call", async () => {
250
+ let capturedInit: RequestInit | undefined;
251
+ fetchStub.callsFake(async (_url: string, init?: RequestInit) => {
252
+ capturedInit = init;
253
+ return makeFetchResponse(200, 'ok', 'text/plain');
254
+ });
255
+
256
+ const fetcher = new WebFetcher({
257
+ requestConfig: { keepalive: true },
258
+ });
259
+ const ok = await fetcher.fetch("https://example.com", async () => true);
260
+ assert.strictEqual(ok, true);
261
+ assert.strictEqual(capturedInit!.keepalive, true);
262
+ });
263
+
264
+ it("htmlToMarkdown trims overly long header text when first space/newline index > 64", () => {
265
+ const fetcher = new WebFetcher();
266
+ const longText = "a".repeat(70) + " rest\nmore";
267
+ const html = `<html><body><p>${longText}</p></body></html>`;
268
+ const md = (fetcher as any)["htmlToMarkdown"](html, "https://example.com");
269
+ assert.ok(!md.startsWith("a".repeat(70)));
270
+ assert.ok(md.includes("rest"));
271
+ });
272
+
273
+ it("htmlToMarkdown leaves short content unchanged", () => {
274
+ const fetcher = new WebFetcher();
275
+ const html = `<html><body><p>short text</p></body></html>`;
276
+ const md = (fetcher as any)["htmlToMarkdown"](html, "https://example.com");
277
+ assert.ok(md.includes("short text"));
278
+ });
279
+
280
+ it("propagates onDocument return value (true/false)", async () => {
281
+ fetchStub.resolves(makeFetchResponse(200, 'ok', 'text/plain'));
282
+ const fetcher = new WebFetcher();
283
+ const yes = await fetcher.fetch("https://example.com/yes", async () => true);
284
+ const no = await fetcher.fetch("https://example.com/no", async () => false);
285
+ assert.strictEqual(yes, true);
286
+ assert.strictEqual(no, false);
287
+ });
288
+ });