ragbench-ai 0.4.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (1336) hide show
  1. ragbench/__init__.py +26 -0
  2. ragbench/chunking/README.md +50 -0
  3. ragbench/chunking/__init__.py +17 -0
  4. ragbench/chunking/chunker.py +507 -0
  5. ragbench/chunking/methods/__init__.py +16 -0
  6. ragbench/chunking/methods/adaptive_chunking/__init__.py +3 -0
  7. ragbench/chunking/methods/adaptive_chunking/metadata.json +10 -0
  8. ragbench/chunking/methods/adaptive_chunking/strategy.py +6 -0
  9. ragbench/chunking/methods/agentic_chunking/__init__.py +3 -0
  10. ragbench/chunking/methods/agentic_chunking/metadata.json +10 -0
  11. ragbench/chunking/methods/agentic_chunking/strategy.py +6 -0
  12. ragbench/chunking/methods/ast_based_chunking/__init__.py +3 -0
  13. ragbench/chunking/methods/ast_based_chunking/metadata.json +12 -0
  14. ragbench/chunking/methods/ast_based_chunking/strategy.py +6 -0
  15. ragbench/chunking/methods/automatic_chunker_selection/__init__.py +3 -0
  16. ragbench/chunking/methods/automatic_chunker_selection/metadata.json +10 -0
  17. ragbench/chunking/methods/automatic_chunker_selection/strategy.py +6 -0
  18. ragbench/chunking/methods/base.py +19 -0
  19. ragbench/chunking/methods/block_based_chunking/__init__.py +3 -0
  20. ragbench/chunking/methods/block_based_chunking/metadata.json +10 -0
  21. ragbench/chunking/methods/block_based_chunking/strategy.py +6 -0
  22. ragbench/chunking/methods/class_level_chunking/__init__.py +3 -0
  23. ragbench/chunking/methods/class_level_chunking/metadata.json +12 -0
  24. ragbench/chunking/methods/class_level_chunking/strategy.py +6 -0
  25. ragbench/chunking/methods/code_block_chunking/__init__.py +3 -0
  26. ragbench/chunking/methods/code_block_chunking/metadata.json +10 -0
  27. ragbench/chunking/methods/code_block_chunking/strategy.py +6 -0
  28. ragbench/chunking/methods/column_based_chunking/__init__.py +3 -0
  29. ragbench/chunking/methods/column_based_chunking/metadata.json +13 -0
  30. ragbench/chunking/methods/column_based_chunking/strategy.py +6 -0
  31. ragbench/chunking/methods/community_based_chunking/__init__.py +3 -0
  32. ragbench/chunking/methods/community_based_chunking/metadata.json +10 -0
  33. ragbench/chunking/methods/community_based_chunking/strategy.py +6 -0
  34. ragbench/chunking/methods/content_type_aware_chunking/__init__.py +3 -0
  35. ragbench/chunking/methods/content_type_aware_chunking/metadata.json +10 -0
  36. ragbench/chunking/methods/content_type_aware_chunking/strategy.py +6 -0
  37. ragbench/chunking/methods/contextual_chunking/__init__.py +3 -0
  38. ragbench/chunking/methods/contextual_chunking/metadata.json +10 -0
  39. ragbench/chunking/methods/contextual_chunking/strategy.py +6 -0
  40. ragbench/chunking/methods/conversation_turn_chunking/__init__.py +3 -0
  41. ragbench/chunking/methods/conversation_turn_chunking/metadata.json +10 -0
  42. ragbench/chunking/methods/conversation_turn_chunking/strategy.py +6 -0
  43. ragbench/chunking/methods/document_section_paragraph_chunking/__init__.py +3 -0
  44. ragbench/chunking/methods/document_section_paragraph_chunking/metadata.json +10 -0
  45. ragbench/chunking/methods/document_section_paragraph_chunking/strategy.py +6 -0
  46. ragbench/chunking/methods/docx_heading_aware_chunking/__init__.py +3 -0
  47. ragbench/chunking/methods/docx_heading_aware_chunking/metadata.json +12 -0
  48. ragbench/chunking/methods/docx_heading_aware_chunking/strategy.py +6 -0
  49. ragbench/chunking/methods/docx_paragraph_chunking/__init__.py +3 -0
  50. ragbench/chunking/methods/docx_paragraph_chunking/metadata.json +14 -0
  51. ragbench/chunking/methods/docx_paragraph_chunking/strategy.py +6 -0
  52. ragbench/chunking/methods/docx_table_aware_chunking/__init__.py +3 -0
  53. ragbench/chunking/methods/docx_table_aware_chunking/metadata.json +12 -0
  54. ragbench/chunking/methods/docx_table_aware_chunking/strategy.py +6 -0
  55. ragbench/chunking/methods/dom_based_chunking/__init__.py +3 -0
  56. ragbench/chunking/methods/dom_based_chunking/metadata.json +13 -0
  57. ragbench/chunking/methods/dom_based_chunking/strategy.py +6 -0
  58. ragbench/chunking/methods/dynamic_chunk_size_chunking/__init__.py +3 -0
  59. ragbench/chunking/methods/dynamic_chunk_size_chunking/metadata.json +10 -0
  60. ragbench/chunking/methods/dynamic_chunk_size_chunking/strategy.py +6 -0
  61. ragbench/chunking/methods/edge_relationship_based_chunking/__init__.py +3 -0
  62. ragbench/chunking/methods/edge_relationship_based_chunking/metadata.json +10 -0
  63. ragbench/chunking/methods/edge_relationship_based_chunking/strategy.py +6 -0
  64. ragbench/chunking/methods/embedding_based_semantic_chunking/__init__.py +3 -0
  65. ragbench/chunking/methods/embedding_based_semantic_chunking/metadata.json +12 -0
  66. ragbench/chunking/methods/embedding_based_semantic_chunking/strategy.py +6 -0
  67. ragbench/chunking/methods/fixed_character_chunking/__init__.py +3 -0
  68. ragbench/chunking/methods/fixed_character_chunking/metadata.json +10 -0
  69. ragbench/chunking/methods/fixed_character_chunking/strategy.py +6 -0
  70. ragbench/chunking/methods/fixed_token_chunking/__init__.py +3 -0
  71. ragbench/chunking/methods/fixed_token_chunking/metadata.json +10 -0
  72. ragbench/chunking/methods/fixed_token_chunking/strategy.py +6 -0
  73. ragbench/chunking/methods/function_level_chunking/__init__.py +3 -0
  74. ragbench/chunking/methods/function_level_chunking/metadata.json +12 -0
  75. ragbench/chunking/methods/function_level_chunking/strategy.py +6 -0
  76. ragbench/chunking/methods/header_row_chunking/__init__.py +3 -0
  77. ragbench/chunking/methods/header_row_chunking/metadata.json +14 -0
  78. ragbench/chunking/methods/header_row_chunking/strategy.py +6 -0
  79. ragbench/chunking/methods/heading_based_chunking/__init__.py +3 -0
  80. ragbench/chunking/methods/heading_based_chunking/metadata.json +10 -0
  81. ragbench/chunking/methods/heading_based_chunking/strategy.py +6 -0
  82. ragbench/chunking/methods/hierarchical_chunking/__init__.py +3 -0
  83. ragbench/chunking/methods/hierarchical_chunking/metadata.json +10 -0
  84. ragbench/chunking/methods/hierarchical_chunking/strategy.py +6 -0
  85. ragbench/chunking/methods/html_element_based_chunking/__init__.py +3 -0
  86. ragbench/chunking/methods/html_element_based_chunking/metadata.json +13 -0
  87. ragbench/chunking/methods/html_element_based_chunking/strategy.py +6 -0
  88. ragbench/chunking/methods/html_section_chunking/__init__.py +3 -0
  89. ragbench/chunking/methods/html_section_chunking/metadata.json +13 -0
  90. ragbench/chunking/methods/html_section_chunking/strategy.py +6 -0
  91. ragbench/chunking/methods/import_dependency_aware_chunking/__init__.py +3 -0
  92. ragbench/chunking/methods/import_dependency_aware_chunking/metadata.json +12 -0
  93. ragbench/chunking/methods/import_dependency_aware_chunking/strategy.py +6 -0
  94. ragbench/chunking/methods/json_array_record_chunking/__init__.py +3 -0
  95. ragbench/chunking/methods/json_array_record_chunking/metadata.json +13 -0
  96. ragbench/chunking/methods/json_array_record_chunking/strategy.py +6 -0
  97. ragbench/chunking/methods/json_nested_object_chunking/__init__.py +3 -0
  98. ragbench/chunking/methods/json_nested_object_chunking/metadata.json +12 -0
  99. ragbench/chunking/methods/json_nested_object_chunking/strategy.py +6 -0
  100. ragbench/chunking/methods/json_object_chunking/__init__.py +3 -0
  101. ragbench/chunking/methods/json_object_chunking/metadata.json +12 -0
  102. ragbench/chunking/methods/json_object_chunking/strategy.py +6 -0
  103. ragbench/chunking/methods/jsonpath_based_chunking/__init__.py +3 -0
  104. ragbench/chunking/methods/jsonpath_based_chunking/metadata.json +12 -0
  105. ragbench/chunking/methods/jsonpath_based_chunking/strategy.py +6 -0
  106. ragbench/chunking/methods/key_value_chunking/__init__.py +3 -0
  107. ragbench/chunking/methods/key_value_chunking/metadata.json +12 -0
  108. ragbench/chunking/methods/key_value_chunking/strategy.py +6 -0
  109. ragbench/chunking/methods/layout_aware_image_chunking/__init__.py +3 -0
  110. ragbench/chunking/methods/layout_aware_image_chunking/metadata.json +16 -0
  111. ragbench/chunking/methods/layout_aware_image_chunking/strategy.py +6 -0
  112. ragbench/chunking/methods/list_aware_chunking/__init__.py +3 -0
  113. ragbench/chunking/methods/list_aware_chunking/metadata.json +10 -0
  114. ragbench/chunking/methods/list_aware_chunking/strategy.py +6 -0
  115. ragbench/chunking/methods/llm_based_semantic_chunking/__init__.py +3 -0
  116. ragbench/chunking/methods/llm_based_semantic_chunking/metadata.json +12 -0
  117. ragbench/chunking/methods/llm_based_semantic_chunking/strategy.py +6 -0
  118. ragbench/chunking/methods/markdown_code_block_aware_chunking/__init__.py +3 -0
  119. ragbench/chunking/methods/markdown_code_block_aware_chunking/metadata.json +12 -0
  120. ragbench/chunking/methods/markdown_code_block_aware_chunking/strategy.py +6 -0
  121. ragbench/chunking/methods/markdown_heading_chunking/__init__.py +3 -0
  122. ragbench/chunking/methods/markdown_heading_chunking/metadata.json +13 -0
  123. ragbench/chunking/methods/markdown_heading_chunking/strategy.py +6 -0
  124. ragbench/chunking/methods/markdown_section_chunking/__init__.py +3 -0
  125. ragbench/chunking/methods/markdown_section_chunking/metadata.json +13 -0
  126. ragbench/chunking/methods/markdown_section_chunking/strategy.py +6 -0
  127. ragbench/chunking/methods/markdown_table_aware_chunking/__init__.py +3 -0
  128. ragbench/chunking/methods/markdown_table_aware_chunking/metadata.json +12 -0
  129. ragbench/chunking/methods/markdown_table_aware_chunking/strategy.py +6 -0
  130. ragbench/chunking/methods/message_based_chunking/__init__.py +3 -0
  131. ragbench/chunking/methods/message_based_chunking/metadata.json +10 -0
  132. ragbench/chunking/methods/message_based_chunking/strategy.py +6 -0
  133. ragbench/chunking/methods/method_level_chunking/__init__.py +3 -0
  134. ragbench/chunking/methods/method_level_chunking/metadata.json +12 -0
  135. ragbench/chunking/methods/method_level_chunking/strategy.py +6 -0
  136. ragbench/chunking/methods/module_level_chunking/__init__.py +3 -0
  137. ragbench/chunking/methods/module_level_chunking/metadata.json +12 -0
  138. ragbench/chunking/methods/module_level_chunking/strategy.py +6 -0
  139. ragbench/chunking/methods/multi_resolution_chunking/__init__.py +3 -0
  140. ragbench/chunking/methods/multi_resolution_chunking/metadata.json +10 -0
  141. ragbench/chunking/methods/multi_resolution_chunking/strategy.py +6 -0
  142. ragbench/chunking/methods/multi_strategy_chunking/__init__.py +3 -0
  143. ragbench/chunking/methods/multi_strategy_chunking/metadata.json +10 -0
  144. ragbench/chunking/methods/multi_strategy_chunking/strategy.py +6 -0
  145. ragbench/chunking/methods/multimodal_video_chunking/__init__.py +3 -0
  146. ragbench/chunking/methods/multimodal_video_chunking/metadata.json +15 -0
  147. ragbench/chunking/methods/multimodal_video_chunking/strategy.py +6 -0
  148. ragbench/chunking/methods/neighborhood_chunking/__init__.py +3 -0
  149. ragbench/chunking/methods/neighborhood_chunking/metadata.json +10 -0
  150. ragbench/chunking/methods/neighborhood_chunking/strategy.py +6 -0
  151. ragbench/chunking/methods/nested_document_chunking/__init__.py +3 -0
  152. ragbench/chunking/methods/nested_document_chunking/metadata.json +12 -0
  153. ragbench/chunking/methods/nested_document_chunking/strategy.py +6 -0
  154. ragbench/chunking/methods/node_based_chunking/__init__.py +3 -0
  155. ragbench/chunking/methods/node_based_chunking/metadata.json +10 -0
  156. ragbench/chunking/methods/node_based_chunking/strategy.py +6 -0
  157. ragbench/chunking/methods/ocr_based_chunking/__init__.py +3 -0
  158. ragbench/chunking/methods/ocr_based_chunking/metadata.json +16 -0
  159. ragbench/chunking/methods/ocr_based_chunking/strategy.py +6 -0
  160. ragbench/chunking/methods/page_based_chunking/__init__.py +3 -0
  161. ragbench/chunking/methods/page_based_chunking/metadata.json +12 -0
  162. ragbench/chunking/methods/page_based_chunking/strategy.py +6 -0
  163. ragbench/chunking/methods/paragraph_chunking/__init__.py +3 -0
  164. ragbench/chunking/methods/paragraph_chunking/metadata.json +10 -0
  165. ragbench/chunking/methods/paragraph_chunking/strategy.py +6 -0
  166. ragbench/chunking/methods/parent_child_chunking/__init__.py +3 -0
  167. ragbench/chunking/methods/parent_child_chunking/metadata.json +10 -0
  168. ragbench/chunking/methods/parent_child_chunking/strategy.py +6 -0
  169. ragbench/chunking/methods/pdf_heading_aware_chunking/__init__.py +3 -0
  170. ragbench/chunking/methods/pdf_heading_aware_chunking/metadata.json +12 -0
  171. ragbench/chunking/methods/pdf_heading_aware_chunking/strategy.py +6 -0
  172. ragbench/chunking/methods/pdf_layout_aware_chunking/__init__.py +3 -0
  173. ragbench/chunking/methods/pdf_layout_aware_chunking/metadata.json +12 -0
  174. ragbench/chunking/methods/pdf_layout_aware_chunking/strategy.py +6 -0
  175. ragbench/chunking/methods/pdf_multi_column_aware_chunking/__init__.py +3 -0
  176. ragbench/chunking/methods/pdf_multi_column_aware_chunking/metadata.json +12 -0
  177. ragbench/chunking/methods/pdf_multi_column_aware_chunking/strategy.py +6 -0
  178. ragbench/chunking/methods/pdf_section_chunking/__init__.py +3 -0
  179. ragbench/chunking/methods/pdf_section_chunking/metadata.json +12 -0
  180. ragbench/chunking/methods/pdf_section_chunking/strategy.py +6 -0
  181. ragbench/chunking/methods/pdf_table_aware_chunking/__init__.py +3 -0
  182. ragbench/chunking/methods/pdf_table_aware_chunking/metadata.json +12 -0
  183. ragbench/chunking/methods/pdf_table_aware_chunking/strategy.py +6 -0
  184. ragbench/chunking/methods/pptx_element_aware_chunking/__init__.py +3 -0
  185. ragbench/chunking/methods/pptx_element_aware_chunking/metadata.json +13 -0
  186. ragbench/chunking/methods/pptx_element_aware_chunking/strategy.py +6 -0
  187. ragbench/chunking/methods/pptx_slide_based_chunking/__init__.py +3 -0
  188. ragbench/chunking/methods/pptx_slide_based_chunking/metadata.json +13 -0
  189. ragbench/chunking/methods/pptx_slide_based_chunking/strategy.py +6 -0
  190. ragbench/chunking/methods/proposition_based_chunking/__init__.py +3 -0
  191. ragbench/chunking/methods/proposition_based_chunking/metadata.json +10 -0
  192. ragbench/chunking/methods/proposition_based_chunking/strategy.py +6 -0
  193. ragbench/chunking/methods/query_aware_chunking/__init__.py +3 -0
  194. ragbench/chunking/methods/query_aware_chunking/metadata.json +10 -0
  195. ragbench/chunking/methods/query_aware_chunking/strategy.py +6 -0
  196. ragbench/chunking/methods/record_based_chunking/__init__.py +3 -0
  197. ragbench/chunking/methods/record_based_chunking/metadata.json +16 -0
  198. ragbench/chunking/methods/record_based_chunking/strategy.py +6 -0
  199. ragbench/chunking/methods/recursive_character_chunking/__init__.py +3 -0
  200. ragbench/chunking/methods/recursive_character_chunking/metadata.json +10 -0
  201. ragbench/chunking/methods/recursive_character_chunking/strategy.py +6 -0
  202. ragbench/chunking/methods/recursive_token_chunking/__init__.py +3 -0
  203. ragbench/chunking/methods/recursive_token_chunking/metadata.json +10 -0
  204. ragbench/chunking/methods/recursive_token_chunking/strategy.py +6 -0
  205. ragbench/chunking/methods/region_based_chunking/__init__.py +3 -0
  206. ragbench/chunking/methods/region_based_chunking/metadata.json +16 -0
  207. ragbench/chunking/methods/region_based_chunking/strategy.py +6 -0
  208. ragbench/chunking/methods/repository_hierarchy_chunking/__init__.py +3 -0
  209. ragbench/chunking/methods/repository_hierarchy_chunking/metadata.json +12 -0
  210. ragbench/chunking/methods/repository_hierarchy_chunking/strategy.py +6 -0
  211. ragbench/chunking/methods/row_based_chunking/__init__.py +3 -0
  212. ragbench/chunking/methods/row_based_chunking/metadata.json +16 -0
  213. ragbench/chunking/methods/row_based_chunking/strategy.py +6 -0
  214. ragbench/chunking/methods/row_group_chunking/__init__.py +3 -0
  215. ragbench/chunking/methods/row_group_chunking/metadata.json +16 -0
  216. ragbench/chunking/methods/row_group_chunking/strategy.py +6 -0
  217. ragbench/chunking/methods/scene_based_chunking/__init__.py +3 -0
  218. ragbench/chunking/methods/scene_based_chunking/metadata.json +15 -0
  219. ragbench/chunking/methods/scene_based_chunking/strategy.py +6 -0
  220. ragbench/chunking/methods/schema_aware_chunking/__init__.py +3 -0
  221. ragbench/chunking/methods/schema_aware_chunking/metadata.json +10 -0
  222. ragbench/chunking/methods/schema_aware_chunking/strategy.py +6 -0
  223. ragbench/chunking/methods/schema_aware_table_chunking/__init__.py +3 -0
  224. ragbench/chunking/methods/schema_aware_table_chunking/metadata.json +10 -0
  225. ragbench/chunking/methods/schema_aware_table_chunking/strategy.py +6 -0
  226. ragbench/chunking/methods/section_based_chunking/__init__.py +3 -0
  227. ragbench/chunking/methods/section_based_chunking/metadata.json +10 -0
  228. ragbench/chunking/methods/section_based_chunking/strategy.py +6 -0
  229. ragbench/chunking/methods/semantic_audio_chunking/__init__.py +3 -0
  230. ragbench/chunking/methods/semantic_audio_chunking/metadata.json +15 -0
  231. ragbench/chunking/methods/semantic_audio_chunking/strategy.py +6 -0
  232. ragbench/chunking/methods/semantic_chunking/__init__.py +3 -0
  233. ragbench/chunking/methods/semantic_chunking/metadata.json +10 -0
  234. ragbench/chunking/methods/semantic_chunking/strategy.py +6 -0
  235. ragbench/chunking/methods/sentence_based_audio_chunking/__init__.py +3 -0
  236. ragbench/chunking/methods/sentence_based_audio_chunking/metadata.json +10 -0
  237. ragbench/chunking/methods/sentence_based_audio_chunking/strategy.py +6 -0
  238. ragbench/chunking/methods/sentence_chunking/__init__.py +3 -0
  239. ragbench/chunking/methods/sentence_chunking/metadata.json +10 -0
  240. ragbench/chunking/methods/sentence_chunking/strategy.py +6 -0
  241. ragbench/chunking/methods/sentence_token_hybrid_chunking/__init__.py +3 -0
  242. ragbench/chunking/methods/sentence_token_hybrid_chunking/metadata.json +10 -0
  243. ragbench/chunking/methods/sentence_token_hybrid_chunking/strategy.py +6 -0
  244. ragbench/chunking/methods/sentence_window_chunking/__init__.py +3 -0
  245. ragbench/chunking/methods/sentence_window_chunking/metadata.json +10 -0
  246. ragbench/chunking/methods/sentence_window_chunking/strategy.py +6 -0
  247. ragbench/chunking/methods/shot_based_chunking/__init__.py +3 -0
  248. ragbench/chunking/methods/shot_based_chunking/metadata.json +15 -0
  249. ragbench/chunking/methods/shot_based_chunking/strategy.py +6 -0
  250. ragbench/chunking/methods/silence_voice_activity_based_chunking/__init__.py +3 -0
  251. ragbench/chunking/methods/silence_voice_activity_based_chunking/metadata.json +15 -0
  252. ragbench/chunking/methods/silence_voice_activity_based_chunking/strategy.py +6 -0
  253. ragbench/chunking/methods/similarity_boundary_chunking/__init__.py +3 -0
  254. ragbench/chunking/methods/similarity_boundary_chunking/metadata.json +10 -0
  255. ragbench/chunking/methods/similarity_boundary_chunking/strategy.py +6 -0
  256. ragbench/chunking/methods/sliding_window_chunking/__init__.py +3 -0
  257. ragbench/chunking/methods/sliding_window_chunking/metadata.json +10 -0
  258. ragbench/chunking/methods/sliding_window_chunking/strategy.py +6 -0
  259. ragbench/chunking/methods/speaker_aware_chunking/__init__.py +3 -0
  260. ragbench/chunking/methods/speaker_aware_chunking/metadata.json +10 -0
  261. ragbench/chunking/methods/speaker_aware_chunking/strategy.py +6 -0
  262. ragbench/chunking/methods/speaker_based_chunking/__init__.py +3 -0
  263. ragbench/chunking/methods/speaker_based_chunking/metadata.json +10 -0
  264. ragbench/chunking/methods/speaker_based_chunking/strategy.py +6 -0
  265. ragbench/chunking/methods/sql_ddl_aware_chunking/__init__.py +3 -0
  266. ragbench/chunking/methods/sql_ddl_aware_chunking/metadata.json +12 -0
  267. ragbench/chunking/methods/sql_ddl_aware_chunking/strategy.py +6 -0
  268. ragbench/chunking/methods/sql_query_result_chunking/__init__.py +3 -0
  269. ragbench/chunking/methods/sql_query_result_chunking/metadata.json +16 -0
  270. ragbench/chunking/methods/sql_query_result_chunking/strategy.py +6 -0
  271. ragbench/chunking/methods/sql_statement_chunking/__init__.py +3 -0
  272. ragbench/chunking/methods/sql_statement_chunking/metadata.json +12 -0
  273. ragbench/chunking/methods/sql_statement_chunking/strategy.py +6 -0
  274. ragbench/chunking/methods/structure_preserving_chunking/__init__.py +3 -0
  275. ragbench/chunking/methods/structure_preserving_chunking/metadata.json +10 -0
  276. ragbench/chunking/methods/structure_preserving_chunking/strategy.py +6 -0
  277. ragbench/chunking/methods/structure_semantic_hybrid_chunking/__init__.py +3 -0
  278. ragbench/chunking/methods/structure_semantic_hybrid_chunking/metadata.json +10 -0
  279. ragbench/chunking/methods/structure_semantic_hybrid_chunking/strategy.py +6 -0
  280. ragbench/chunking/methods/subgraph_chunking/__init__.py +3 -0
  281. ragbench/chunking/methods/subgraph_chunking/metadata.json +10 -0
  282. ragbench/chunking/methods/subgraph_chunking/strategy.py +6 -0
  283. ragbench/chunking/methods/table_based_chunking/__init__.py +3 -0
  284. ragbench/chunking/methods/table_based_chunking/metadata.json +14 -0
  285. ragbench/chunking/methods/table_based_chunking/strategy.py +6 -0
  286. ragbench/chunking/methods/table_level_chunking/__init__.py +3 -0
  287. ragbench/chunking/methods/table_level_chunking/metadata.json +14 -0
  288. ragbench/chunking/methods/table_level_chunking/strategy.py +6 -0
  289. ragbench/chunking/methods/table_region_chunking/__init__.py +3 -0
  290. ragbench/chunking/methods/table_region_chunking/metadata.json +12 -0
  291. ragbench/chunking/methods/table_region_chunking/strategy.py +6 -0
  292. ragbench/chunking/methods/thread_based_chunking/__init__.py +3 -0
  293. ragbench/chunking/methods/thread_based_chunking/metadata.json +10 -0
  294. ragbench/chunking/methods/thread_based_chunking/strategy.py +6 -0
  295. ragbench/chunking/methods/time_based_chunking/__init__.py +3 -0
  296. ragbench/chunking/methods/time_based_chunking/metadata.json +10 -0
  297. ragbench/chunking/methods/time_based_chunking/strategy.py +6 -0
  298. ragbench/chunking/methods/time_window_conversation_chunking/__init__.py +3 -0
  299. ragbench/chunking/methods/time_window_conversation_chunking/metadata.json +10 -0
  300. ragbench/chunking/methods/time_window_conversation_chunking/strategy.py +6 -0
  301. ragbench/chunking/methods/time_window_video_chunking/__init__.py +3 -0
  302. ragbench/chunking/methods/time_window_video_chunking/metadata.json +10 -0
  303. ragbench/chunking/methods/time_window_video_chunking/strategy.py +6 -0
  304. ragbench/chunking/methods/topic_based_chunking/__init__.py +3 -0
  305. ragbench/chunking/methods/topic_based_chunking/metadata.json +10 -0
  306. ragbench/chunking/methods/topic_based_chunking/strategy.py +6 -0
  307. ragbench/chunking/methods/transcript_based_chunking/__init__.py +3 -0
  308. ragbench/chunking/methods/transcript_based_chunking/metadata.json +10 -0
  309. ragbench/chunking/methods/transcript_based_chunking/strategy.py +6 -0
  310. ragbench/chunking/methods/transcript_based_video_chunking/__init__.py +3 -0
  311. ragbench/chunking/methods/transcript_based_video_chunking/metadata.json +10 -0
  312. ragbench/chunking/methods/transcript_based_video_chunking/strategy.py +6 -0
  313. ragbench/chunking/methods/vision_semantic_region_chunking/__init__.py +3 -0
  314. ragbench/chunking/methods/vision_semantic_region_chunking/metadata.json +16 -0
  315. ragbench/chunking/methods/vision_semantic_region_chunking/strategy.py +6 -0
  316. ragbench/chunking/methods/xml_tree_based_chunking/__init__.py +3 -0
  317. ragbench/chunking/methods/xml_tree_based_chunking/metadata.json +12 -0
  318. ragbench/chunking/methods/xml_tree_based_chunking/strategy.py +6 -0
  319. ragbench/chunking/methods/xpath_based_chunking/__init__.py +3 -0
  320. ragbench/chunking/methods/xpath_based_chunking/metadata.json +12 -0
  321. ragbench/chunking/methods/xpath_based_chunking/strategy.py +6 -0
  322. ragbench/chunking/methods/yaml_structure_based_chunking/__init__.py +3 -0
  323. ragbench/chunking/methods/yaml_structure_based_chunking/metadata.json +13 -0
  324. ragbench/chunking/methods/yaml_structure_based_chunking/strategy.py +6 -0
  325. ragbench/chunking/models.py +13 -0
  326. ragbench/chunking/registry.py +90 -0
  327. ragbench/cli.py +387 -0
  328. ragbench/connectors/__init__.py +12 -0
  329. ragbench/connectors/aac/connector.py +31 -0
  330. ragbench/connectors/aac/metadata.json +8 -0
  331. ragbench/connectors/aac/requirements.txt +1 -0
  332. ragbench/connectors/arangodb/connector.py +83 -0
  333. ragbench/connectors/arangodb/metadata.json +8 -0
  334. ragbench/connectors/arangodb/requirements.txt +1 -0
  335. ragbench/connectors/avi/connector.py +30 -0
  336. ragbench/connectors/avi/metadata.json +8 -0
  337. ragbench/connectors/avi/requirements.txt +1 -0
  338. ragbench/connectors/avro/connector.py +32 -0
  339. ragbench/connectors/avro/metadata.json +8 -0
  340. ragbench/connectors/avro/requirements.txt +1 -0
  341. ragbench/connectors/azure_blob/connector.py +200 -0
  342. ragbench/connectors/azure_blob/metadata.json +17 -0
  343. ragbench/connectors/azure_blob/requirements.txt +1 -0
  344. ragbench/connectors/bigquery/connector.py +91 -0
  345. ragbench/connectors/bigquery/metadata.json +8 -0
  346. ragbench/connectors/bigquery/requirements.txt +1 -0
  347. ragbench/connectors/bmp/connector.py +30 -0
  348. ragbench/connectors/bmp/metadata.json +8 -0
  349. ragbench/connectors/bmp/requirements.txt +1 -0
  350. ragbench/connectors/cassandra/connector.py +133 -0
  351. ragbench/connectors/cassandra/metadata.json +8 -0
  352. ragbench/connectors/cassandra/requirements.txt +1 -0
  353. ragbench/connectors/clickhouse/connector.py +112 -0
  354. ragbench/connectors/clickhouse/metadata.json +8 -0
  355. ragbench/connectors/clickhouse/requirements.txt +1 -0
  356. ragbench/connectors/csv/connector.py +30 -0
  357. ragbench/connectors/csv/metadata.json +8 -0
  358. ragbench/connectors/csv/requirements.txt +1 -0
  359. ragbench/connectors/databricks/connector.py +94 -0
  360. ragbench/connectors/databricks/metadata.json +8 -0
  361. ragbench/connectors/databricks/requirements.txt +1 -0
  362. ragbench/connectors/docx/connector.py +31 -0
  363. ragbench/connectors/docx/metadata.json +8 -0
  364. ragbench/connectors/docx/requirements.txt +1 -0
  365. ragbench/connectors/duckdb/connector.py +93 -0
  366. ragbench/connectors/duckdb/metadata.json +8 -0
  367. ragbench/connectors/duckdb/requirements.txt +1 -0
  368. ragbench/connectors/dynamodb/connector.py +79 -0
  369. ragbench/connectors/dynamodb/metadata.json +8 -0
  370. ragbench/connectors/dynamodb/requirements.txt +1 -0
  371. ragbench/connectors/elasticsearch/connector.py +92 -0
  372. ragbench/connectors/elasticsearch/metadata.json +8 -0
  373. ragbench/connectors/elasticsearch/requirements.txt +1 -0
  374. ragbench/connectors/excel/connector.py +30 -0
  375. ragbench/connectors/excel/metadata.json +9 -0
  376. ragbench/connectors/excel/requirements.txt +1 -0
  377. ragbench/connectors/feather/connector.py +30 -0
  378. ragbench/connectors/feather/metadata.json +8 -0
  379. ragbench/connectors/feather/requirements.txt +2 -0
  380. ragbench/connectors/flac/connector.py +31 -0
  381. ragbench/connectors/flac/metadata.json +8 -0
  382. ragbench/connectors/flac/requirements.txt +1 -0
  383. ragbench/connectors/gcs/connector.py +186 -0
  384. ragbench/connectors/gcs/metadata.json +15 -0
  385. ragbench/connectors/gcs/requirements.txt +2 -0
  386. ragbench/connectors/google_drive/connector.py +276 -0
  387. ragbench/connectors/google_drive/metadata.json +16 -0
  388. ragbench/connectors/google_drive/requirements.txt +3 -0
  389. ragbench/connectors/gzip/connector.py +30 -0
  390. ragbench/connectors/gzip/metadata.json +9 -0
  391. ragbench/connectors/gzip/requirements.txt +1 -0
  392. ragbench/connectors/html/connector.py +32 -0
  393. ragbench/connectors/html/metadata.json +9 -0
  394. ragbench/connectors/html/requirements.txt +1 -0
  395. ragbench/connectors/jira/connector.py +212 -0
  396. ragbench/connectors/jira/metadata.json +17 -0
  397. ragbench/connectors/jira/requirements.txt +1 -0
  398. ragbench/connectors/jpeg/connector.py +30 -0
  399. ragbench/connectors/jpeg/metadata.json +9 -0
  400. ragbench/connectors/jpeg/requirements.txt +1 -0
  401. ragbench/connectors/json/connector.py +35 -0
  402. ragbench/connectors/json/metadata.json +8 -0
  403. ragbench/connectors/json/requirements.txt +1 -0
  404. ragbench/connectors/jsonl/connector.py +30 -0
  405. ragbench/connectors/jsonl/metadata.json +8 -0
  406. ragbench/connectors/jsonl/requirements.txt +1 -0
  407. ragbench/connectors/kafka/connector.py +260 -0
  408. ragbench/connectors/kafka/metadata.json +17 -0
  409. ragbench/connectors/kafka/requirements.txt +1 -0
  410. ragbench/connectors/mariadb/connector.py +135 -0
  411. ragbench/connectors/mariadb/metadata.json +8 -0
  412. ragbench/connectors/mariadb/requirements.txt +2 -0
  413. ragbench/connectors/markdown/connector.py +32 -0
  414. ragbench/connectors/markdown/metadata.json +8 -0
  415. ragbench/connectors/markdown/requirements.txt +1 -0
  416. ragbench/connectors/memcached/connector.py +68 -0
  417. ragbench/connectors/memcached/metadata.json +8 -0
  418. ragbench/connectors/memcached/requirements.txt +1 -0
  419. ragbench/connectors/milvus/connector.py +80 -0
  420. ragbench/connectors/milvus/metadata.json +8 -0
  421. ragbench/connectors/milvus/requirements.txt +1 -0
  422. ragbench/connectors/minio/connector.py +188 -0
  423. ragbench/connectors/minio/metadata.json +18 -0
  424. ragbench/connectors/minio/requirements.txt +1 -0
  425. ragbench/connectors/mkv/connector.py +30 -0
  426. ragbench/connectors/mkv/metadata.json +8 -0
  427. ragbench/connectors/mkv/requirements.txt +1 -0
  428. ragbench/connectors/mongodb/connector.py +96 -0
  429. ragbench/connectors/mongodb/metadata.json +8 -0
  430. ragbench/connectors/mongodb/requirements.txt +1 -0
  431. ragbench/connectors/mov/connector.py +30 -0
  432. ragbench/connectors/mov/metadata.json +8 -0
  433. ragbench/connectors/mov/requirements.txt +1 -0
  434. ragbench/connectors/mp3/connector.py +31 -0
  435. ragbench/connectors/mp3/metadata.json +8 -0
  436. ragbench/connectors/mp3/requirements.txt +1 -0
  437. ragbench/connectors/mp4/connector.py +30 -0
  438. ragbench/connectors/mp4/metadata.json +8 -0
  439. ragbench/connectors/mp4/requirements.txt +1 -0
  440. ragbench/connectors/mssql/connector.py +134 -0
  441. ragbench/connectors/mssql/metadata.json +8 -0
  442. ragbench/connectors/mssql/requirements.txt +2 -0
  443. ragbench/connectors/mysql/connector.py +136 -0
  444. ragbench/connectors/mysql/metadata.json +8 -0
  445. ragbench/connectors/mysql/requirements.txt +2 -0
  446. ragbench/connectors/neo4j/connector.py +95 -0
  447. ragbench/connectors/neo4j/metadata.json +8 -0
  448. ragbench/connectors/neo4j/requirements.txt +1 -0
  449. ragbench/connectors/notion/connector.py +268 -0
  450. ragbench/connectors/notion/metadata.json +16 -0
  451. ragbench/connectors/notion/requirements.txt +1 -0
  452. ragbench/connectors/odp/connector.py +30 -0
  453. ragbench/connectors/odp/metadata.json +8 -0
  454. ragbench/connectors/odp/requirements.txt +1 -0
  455. ragbench/connectors/ods/connector.py +30 -0
  456. ragbench/connectors/ods/metadata.json +8 -0
  457. ragbench/connectors/ods/requirements.txt +2 -0
  458. ragbench/connectors/odt/connector.py +30 -0
  459. ragbench/connectors/odt/metadata.json +8 -0
  460. ragbench/connectors/odt/requirements.txt +1 -0
  461. ragbench/connectors/oracle/connector.py +139 -0
  462. ragbench/connectors/oracle/metadata.json +8 -0
  463. ragbench/connectors/oracle/requirements.txt +2 -0
  464. ragbench/connectors/orc/connector.py +31 -0
  465. ragbench/connectors/orc/metadata.json +8 -0
  466. ragbench/connectors/orc/requirements.txt +2 -0
  467. ragbench/connectors/parquet/connector.py +31 -0
  468. ragbench/connectors/parquet/metadata.json +8 -0
  469. ragbench/connectors/parquet/requirements.txt +3 -0
  470. ragbench/connectors/pdf_auto/connector.py +83 -0
  471. ragbench/connectors/pdf_auto/metadata.json +8 -0
  472. ragbench/connectors/pdf_auto/requirements.txt +4 -0
  473. ragbench/connectors/pdf_hybrid/connector.py +84 -0
  474. ragbench/connectors/pdf_hybrid/metadata.json +6 -0
  475. ragbench/connectors/pdf_hybrid/requirements.txt +4 -0
  476. ragbench/connectors/pdf_scanned/connector.py +56 -0
  477. ragbench/connectors/pdf_scanned/metadata.json +6 -0
  478. ragbench/connectors/pdf_scanned/requirements.txt +3 -0
  479. ragbench/connectors/pdf_text/connector.py +33 -0
  480. ragbench/connectors/pdf_text/metadata.json +6 -0
  481. ragbench/connectors/pdf_text/requirements.txt +1 -0
  482. ragbench/connectors/pinecone/connector.py +74 -0
  483. ragbench/connectors/pinecone/metadata.json +8 -0
  484. ragbench/connectors/pinecone/requirements.txt +1 -0
  485. ragbench/connectors/png/connector.py +30 -0
  486. ragbench/connectors/png/metadata.json +8 -0
  487. ragbench/connectors/png/requirements.txt +1 -0
  488. ragbench/connectors/postgresql/connector.py +161 -0
  489. ragbench/connectors/postgresql/metadata.json +8 -0
  490. ragbench/connectors/postgresql/requirements.txt +2 -0
  491. ragbench/connectors/pptx/connector.py +36 -0
  492. ragbench/connectors/pptx/metadata.json +8 -0
  493. ragbench/connectors/pptx/requirements.txt +1 -0
  494. ragbench/connectors/presto/connector.py +103 -0
  495. ragbench/connectors/presto/metadata.json +8 -0
  496. ragbench/connectors/presto/requirements.txt +1 -0
  497. ragbench/connectors/qdrant/connector.py +70 -0
  498. ragbench/connectors/qdrant/metadata.json +8 -0
  499. ragbench/connectors/qdrant/requirements.txt +1 -0
  500. ragbench/connectors/rabbitmq/connector.py +232 -0
  501. ragbench/connectors/rabbitmq/metadata.json +17 -0
  502. ragbench/connectors/rabbitmq/requirements.txt +1 -0
  503. ragbench/connectors/redis/connector.py +97 -0
  504. ragbench/connectors/redis/metadata.json +8 -0
  505. ragbench/connectors/redis/requirements.txt +1 -0
  506. ragbench/connectors/redshift/connector.py +112 -0
  507. ragbench/connectors/redshift/metadata.json +8 -0
  508. ragbench/connectors/redshift/requirements.txt +1 -0
  509. ragbench/connectors/rtf/connector.py +31 -0
  510. ragbench/connectors/rtf/metadata.json +8 -0
  511. ragbench/connectors/rtf/requirements.txt +1 -0
  512. ragbench/connectors/s3/connector.py +202 -0
  513. ragbench/connectors/s3/metadata.json +17 -0
  514. ragbench/connectors/s3/requirements.txt +1 -0
  515. ragbench/connectors/sevenz/connector.py +30 -0
  516. ragbench/connectors/sevenz/metadata.json +8 -0
  517. ragbench/connectors/sevenz/requirements.txt +1 -0
  518. ragbench/connectors/sitemap/connector.py +279 -0
  519. ragbench/connectors/sitemap/metadata.json +15 -0
  520. ragbench/connectors/sitemap/requirements.txt +3 -0
  521. ragbench/connectors/slack/connector.py +219 -0
  522. ragbench/connectors/slack/metadata.json +16 -0
  523. ragbench/connectors/slack/requirements.txt +1 -0
  524. ragbench/connectors/snowflake/connector.py +108 -0
  525. ragbench/connectors/snowflake/metadata.json +8 -0
  526. ragbench/connectors/snowflake/requirements.txt +1 -0
  527. ragbench/connectors/solr/connector.py +70 -0
  528. ragbench/connectors/solr/metadata.json +8 -0
  529. ragbench/connectors/solr/requirements.txt +1 -0
  530. ragbench/connectors/sqlite/connector.py +112 -0
  531. ragbench/connectors/sqlite/metadata.json +9 -0
  532. ragbench/connectors/sqlite/requirements.txt +1 -0
  533. ragbench/connectors/supabase/connector.py +88 -0
  534. ragbench/connectors/supabase/metadata.json +8 -0
  535. ragbench/connectors/supabase/requirements.txt +1 -0
  536. ragbench/connectors/tar/connector.py +30 -0
  537. ragbench/connectors/tar/metadata.json +8 -0
  538. ragbench/connectors/tar/requirements.txt +1 -0
  539. ragbench/connectors/tiff/connector.py +30 -0
  540. ragbench/connectors/tiff/metadata.json +9 -0
  541. ragbench/connectors/tiff/requirements.txt +1 -0
  542. ragbench/connectors/trino/connector.py +103 -0
  543. ragbench/connectors/trino/metadata.json +8 -0
  544. ragbench/connectors/trino/requirements.txt +1 -0
  545. ragbench/connectors/tsv/connector.py +30 -0
  546. ragbench/connectors/tsv/metadata.json +8 -0
  547. ragbench/connectors/tsv/requirements.txt +1 -0
  548. ragbench/connectors/txt/connector.py +30 -0
  549. ragbench/connectors/txt/metadata.json +8 -0
  550. ragbench/connectors/txt/requirements.txt +0 -0
  551. ragbench/connectors/wav/connector.py +31 -0
  552. ragbench/connectors/wav/metadata.json +8 -0
  553. ragbench/connectors/wav/requirements.txt +1 -0
  554. ragbench/connectors/weaviate/connector.py +77 -0
  555. ragbench/connectors/weaviate/metadata.json +8 -0
  556. ragbench/connectors/weaviate/requirements.txt +1 -0
  557. ragbench/connectors/web_scraper/connector.py +261 -0
  558. ragbench/connectors/web_scraper/metadata.json +16 -0
  559. ragbench/connectors/web_scraper/requirements.txt +3 -0
  560. ragbench/connectors/webp/connector.py +30 -0
  561. ragbench/connectors/webp/metadata.json +8 -0
  562. ragbench/connectors/webp/requirements.txt +1 -0
  563. ragbench/connectors/xlsx/connector.py +38 -0
  564. ragbench/connectors/xlsx/metadata.json +8 -0
  565. ragbench/connectors/xlsx/requirements.txt +1 -0
  566. ragbench/connectors/xml/connector.py +35 -0
  567. ragbench/connectors/xml/metadata.json +8 -0
  568. ragbench/connectors/xml/requirements.txt +1 -0
  569. ragbench/connectors/yaml/connector.py +31 -0
  570. ragbench/connectors/yaml/metadata.json +9 -0
  571. ragbench/connectors/yaml/requirements.txt +1 -0
  572. ragbench/connectors/zip/connector.py +30 -0
  573. ragbench/connectors/zip/metadata.json +8 -0
  574. ragbench/connectors/zip/requirements.txt +1 -0
  575. ragbench/core/__init__.py +13 -0
  576. ragbench/core/base_connector.py +32 -0
  577. ragbench/core/connectors/local_structured/avro_connector.py +32 -0
  578. ragbench/core/connectors/local_unstructured/pdf_hybrid_connector.py +31 -0
  579. ragbench/core/connectors/local_unstructured/tiff_connector.py +30 -0
  580. ragbench/core/db_connector.py +104 -0
  581. ragbench/core/document/__init__.py +4 -0
  582. ragbench/core/document/document_model.py +27 -0
  583. ragbench/core/loader.py +269 -0
  584. ragbench/core/plugin_manager.py +75 -0
  585. ragbench/core/registry.py +473 -0
  586. ragbench/core/utilities.py +9 -0
  587. ragbench/embedding/README.md +144 -0
  588. ragbench/embedding/__init__.py +21 -0
  589. ragbench/embedding/cache.py +72 -0
  590. ragbench/embedding/embedder.py +139 -0
  591. ragbench/embedding/env.py +78 -0
  592. ragbench/embedding/errors.py +47 -0
  593. ragbench/embedding/methods/__init__.py +4 -0
  594. ragbench/embedding/methods/_http.py +148 -0
  595. ragbench/embedding/methods/_local.py +81 -0
  596. ragbench/embedding/methods/_text.py +25 -0
  597. ragbench/embedding/methods/_wrapper.py +59 -0
  598. ragbench/embedding/methods/arctic/__init__.py +3 -0
  599. ragbench/embedding/methods/arctic/embedder.py +7 -0
  600. ragbench/embedding/methods/arctic/metadata.json +33 -0
  601. ragbench/embedding/methods/arctic/requirements.txt +1 -0
  602. ragbench/embedding/methods/audio_embedder/__init__.py +3 -0
  603. ragbench/embedding/methods/audio_embedder/embedder.py +38 -0
  604. ragbench/embedding/methods/audio_embedder/metadata.json +32 -0
  605. ragbench/embedding/methods/audio_embedder/requirements.txt +3 -0
  606. ragbench/embedding/methods/azure_openai/__init__.py +3 -0
  607. ragbench/embedding/methods/azure_openai/embedder.py +20 -0
  608. ragbench/embedding/methods/azure_openai/metadata.json +42 -0
  609. ragbench/embedding/methods/azure_openai/requirements.txt +0 -0
  610. ragbench/embedding/methods/base.py +303 -0
  611. ragbench/embedding/methods/bedrock/__init__.py +3 -0
  612. ragbench/embedding/methods/bedrock/embedder.py +51 -0
  613. ragbench/embedding/methods/bedrock/metadata.json +37 -0
  614. ragbench/embedding/methods/bedrock/requirements.txt +1 -0
  615. ragbench/embedding/methods/bge/__init__.py +3 -0
  616. ragbench/embedding/methods/bge/embedder.py +7 -0
  617. ragbench/embedding/methods/bge/metadata.json +33 -0
  618. ragbench/embedding/methods/bge/requirements.txt +1 -0
  619. ragbench/embedding/methods/bge_m3/__init__.py +3 -0
  620. ragbench/embedding/methods/bge_m3/embedder.py +34 -0
  621. ragbench/embedding/methods/bge_m3/metadata.json +39 -0
  622. ragbench/embedding/methods/bge_m3/requirements.txt +1 -0
  623. ragbench/embedding/methods/biomedical/__init__.py +3 -0
  624. ragbench/embedding/methods/biomedical/embedder.py +7 -0
  625. ragbench/embedding/methods/biomedical/metadata.json +33 -0
  626. ragbench/embedding/methods/biomedical/requirements.txt +1 -0
  627. ragbench/embedding/methods/bm25/__init__.py +3 -0
  628. ragbench/embedding/methods/bm25/embedder.py +45 -0
  629. ragbench/embedding/methods/bm25/metadata.json +26 -0
  630. ragbench/embedding/methods/bm25/requirements.txt +0 -0
  631. ragbench/embedding/methods/clap/__init__.py +3 -0
  632. ragbench/embedding/methods/clap/embedder.py +39 -0
  633. ragbench/embedding/methods/clap/metadata.json +29 -0
  634. ragbench/embedding/methods/clap/requirements.txt +3 -0
  635. ragbench/embedding/methods/clip/__init__.py +3 -0
  636. ragbench/embedding/methods/clip/embedder.py +34 -0
  637. ragbench/embedding/methods/clip/metadata.json +28 -0
  638. ragbench/embedding/methods/clip/requirements.txt +2 -0
  639. ragbench/embedding/methods/code_api/__init__.py +3 -0
  640. ragbench/embedding/methods/code_api/embedder.py +17 -0
  641. ragbench/embedding/methods/code_api/metadata.json +38 -0
  642. ragbench/embedding/methods/code_api/requirements.txt +0 -0
  643. ragbench/embedding/methods/code_local/__init__.py +3 -0
  644. ragbench/embedding/methods/code_local/embedder.py +15 -0
  645. ragbench/embedding/methods/code_local/metadata.json +33 -0
  646. ragbench/embedding/methods/code_local/requirements.txt +1 -0
  647. ragbench/embedding/methods/cohere/__init__.py +3 -0
  648. ragbench/embedding/methods/cohere/embedder.py +16 -0
  649. ragbench/embedding/methods/cohere/metadata.json +38 -0
  650. ragbench/embedding/methods/cohere/requirements.txt +0 -0
  651. ragbench/embedding/methods/colbert/__init__.py +3 -0
  652. ragbench/embedding/methods/colbert/embedder.py +25 -0
  653. ragbench/embedding/methods/colbert/metadata.json +23 -0
  654. ragbench/embedding/methods/colbert/requirements.txt +1 -0
  655. ragbench/embedding/methods/colpali/__init__.py +3 -0
  656. ragbench/embedding/methods/colpali/embedder.py +44 -0
  657. ragbench/embedding/methods/colpali/metadata.json +28 -0
  658. ragbench/embedding/methods/colpali/requirements.txt +3 -0
  659. ragbench/embedding/methods/contextual_prefix/__init__.py +3 -0
  660. ragbench/embedding/methods/contextual_prefix/embedder.py +55 -0
  661. ragbench/embedding/methods/contextual_prefix/metadata.json +36 -0
  662. ragbench/embedding/methods/contextual_prefix/requirements.txt +0 -0
  663. ragbench/embedding/methods/custom_callable/__init__.py +3 -0
  664. ragbench/embedding/methods/custom_callable/embedder.py +22 -0
  665. ragbench/embedding/methods/custom_callable/metadata.json +25 -0
  666. ragbench/embedding/methods/custom_callable/requirements.txt +0 -0
  667. ragbench/embedding/methods/custom_hf/__init__.py +3 -0
  668. ragbench/embedding/methods/custom_hf/embedder.py +11 -0
  669. ragbench/embedding/methods/custom_hf/metadata.json +37 -0
  670. ragbench/embedding/methods/custom_hf/requirements.txt +2 -0
  671. ragbench/embedding/methods/doc2vec/__init__.py +3 -0
  672. ragbench/embedding/methods/doc2vec/embedder.py +27 -0
  673. ragbench/embedding/methods/doc2vec/metadata.json +26 -0
  674. ragbench/embedding/methods/doc2vec/requirements.txt +1 -0
  675. ragbench/embedding/methods/e5/__init__.py +3 -0
  676. ragbench/embedding/methods/e5/embedder.py +7 -0
  677. ragbench/embedding/methods/e5/metadata.json +33 -0
  678. ragbench/embedding/methods/e5/requirements.txt +1 -0
  679. ragbench/embedding/methods/ensemble_concat/__init__.py +3 -0
  680. ragbench/embedding/methods/ensemble_concat/embedder.py +36 -0
  681. ragbench/embedding/methods/ensemble_concat/metadata.json +28 -0
  682. ragbench/embedding/methods/ensemble_concat/requirements.txt +0 -0
  683. ragbench/embedding/methods/fasttext/__init__.py +3 -0
  684. ragbench/embedding/methods/fasttext/embedder.py +26 -0
  685. ragbench/embedding/methods/fasttext/metadata.json +23 -0
  686. ragbench/embedding/methods/fasttext/requirements.txt +1 -0
  687. ragbench/embedding/methods/finance/__init__.py +3 -0
  688. ragbench/embedding/methods/finance/embedder.py +7 -0
  689. ragbench/embedding/methods/finance/metadata.json +33 -0
  690. ragbench/embedding/methods/finance/requirements.txt +1 -0
  691. ragbench/embedding/methods/fusion/__init__.py +3 -0
  692. ragbench/embedding/methods/fusion/embedder.py +26 -0
  693. ragbench/embedding/methods/fusion/metadata.json +28 -0
  694. ragbench/embedding/methods/fusion/requirements.txt +0 -0
  695. ragbench/embedding/methods/glove/__init__.py +3 -0
  696. ragbench/embedding/methods/glove/embedder.py +70 -0
  697. ragbench/embedding/methods/glove/metadata.json +23 -0
  698. ragbench/embedding/methods/glove/requirements.txt +0 -0
  699. ragbench/embedding/methods/google_gemini/__init__.py +3 -0
  700. ragbench/embedding/methods/google_gemini/embedder.py +26 -0
  701. ragbench/embedding/methods/google_gemini/metadata.json +42 -0
  702. ragbench/embedding/methods/google_gemini/requirements.txt +0 -0
  703. ragbench/embedding/methods/google_vertex/__init__.py +3 -0
  704. ragbench/embedding/methods/google_vertex/embedder.py +28 -0
  705. ragbench/embedding/methods/google_vertex/metadata.json +46 -0
  706. ragbench/embedding/methods/google_vertex/requirements.txt +0 -0
  707. ragbench/embedding/methods/gte/__init__.py +3 -0
  708. ragbench/embedding/methods/gte/embedder.py +7 -0
  709. ragbench/embedding/methods/gte/metadata.json +33 -0
  710. ragbench/embedding/methods/gte/requirements.txt +1 -0
  711. ragbench/embedding/methods/hash_baseline/__init__.py +3 -0
  712. ragbench/embedding/methods/hash_baseline/embedder.py +39 -0
  713. ragbench/embedding/methods/hash_baseline/metadata.json +16 -0
  714. ragbench/embedding/methods/hash_baseline/requirements.txt +0 -0
  715. ragbench/embedding/methods/hashing_sparse/__init__.py +3 -0
  716. ragbench/embedding/methods/hashing_sparse/embedder.py +24 -0
  717. ragbench/embedding/methods/hashing_sparse/metadata.json +26 -0
  718. ragbench/embedding/methods/hashing_sparse/requirements.txt +0 -0
  719. ragbench/embedding/methods/hashing_vectorizer/__init__.py +3 -0
  720. ragbench/embedding/methods/hashing_vectorizer/embedder.py +35 -0
  721. ragbench/embedding/methods/hashing_vectorizer/metadata.json +33 -0
  722. ragbench/embedding/methods/hashing_vectorizer/requirements.txt +0 -0
  723. ragbench/embedding/methods/hf_transformers/__init__.py +3 -0
  724. ragbench/embedding/methods/hf_transformers/embedder.py +5 -0
  725. ragbench/embedding/methods/hf_transformers/metadata.json +43 -0
  726. ragbench/embedding/methods/hf_transformers/requirements.txt +2 -0
  727. ragbench/embedding/methods/image_embedder/__init__.py +3 -0
  728. ragbench/embedding/methods/image_embedder/embedder.py +36 -0
  729. ragbench/embedding/methods/image_embedder/metadata.json +30 -0
  730. ragbench/embedding/methods/image_embedder/requirements.txt +3 -0
  731. ragbench/embedding/methods/jina/__init__.py +3 -0
  732. ragbench/embedding/methods/jina/embedder.py +14 -0
  733. ragbench/embedding/methods/jina/metadata.json +39 -0
  734. ragbench/embedding/methods/jina/requirements.txt +0 -0
  735. ragbench/embedding/methods/l2_normalize/__init__.py +3 -0
  736. ragbench/embedding/methods/l2_normalize/embedder.py +7 -0
  737. ragbench/embedding/methods/l2_normalize/metadata.json +27 -0
  738. ragbench/embedding/methods/l2_normalize/requirements.txt +0 -0
  739. ragbench/embedding/methods/late_chunking/__init__.py +3 -0
  740. ragbench/embedding/methods/late_chunking/embedder.py +89 -0
  741. ragbench/embedding/methods/late_chunking/metadata.json +31 -0
  742. ragbench/embedding/methods/late_chunking/requirements.txt +2 -0
  743. ragbench/embedding/methods/legal/__init__.py +3 -0
  744. ragbench/embedding/methods/legal/embedder.py +5 -0
  745. ragbench/embedding/methods/legal/metadata.json +39 -0
  746. ragbench/embedding/methods/legal/requirements.txt +2 -0
  747. ragbench/embedding/methods/llamacpp/__init__.py +3 -0
  748. ragbench/embedding/methods/llamacpp/embedder.py +11 -0
  749. ragbench/embedding/methods/llamacpp/metadata.json +39 -0
  750. ragbench/embedding/methods/llamacpp/requirements.txt +0 -0
  751. ragbench/embedding/methods/matryoshka/__init__.py +3 -0
  752. ragbench/embedding/methods/matryoshka/embedder.py +24 -0
  753. ragbench/embedding/methods/matryoshka/metadata.json +31 -0
  754. ragbench/embedding/methods/matryoshka/requirements.txt +0 -0
  755. ragbench/embedding/methods/minilm/__init__.py +3 -0
  756. ragbench/embedding/methods/minilm/embedder.py +7 -0
  757. ragbench/embedding/methods/minilm/metadata.json +33 -0
  758. ragbench/embedding/methods/minilm/requirements.txt +1 -0
  759. ragbench/embedding/methods/mistral/__init__.py +3 -0
  760. ragbench/embedding/methods/mistral/embedder.py +10 -0
  761. ragbench/embedding/methods/mistral/metadata.json +36 -0
  762. ragbench/embedding/methods/mistral/requirements.txt +0 -0
  763. ragbench/embedding/methods/multilingual_e5/__init__.py +3 -0
  764. ragbench/embedding/methods/multilingual_e5/embedder.py +7 -0
  765. ragbench/embedding/methods/multilingual_e5/metadata.json +35 -0
  766. ragbench/embedding/methods/multilingual_e5/requirements.txt +1 -0
  767. ragbench/embedding/methods/node2vec/__init__.py +3 -0
  768. ragbench/embedding/methods/node2vec/embedder.py +122 -0
  769. ragbench/embedding/methods/node2vec/metadata.json +45 -0
  770. ragbench/embedding/methods/node2vec/requirements.txt +1 -0
  771. ragbench/embedding/methods/nomic/__init__.py +3 -0
  772. ragbench/embedding/methods/nomic/embedder.py +11 -0
  773. ragbench/embedding/methods/nomic/metadata.json +33 -0
  774. ragbench/embedding/methods/nomic/requirements.txt +1 -0
  775. ragbench/embedding/methods/numeric_features/__init__.py +3 -0
  776. ragbench/embedding/methods/numeric_features/embedder.py +60 -0
  777. ragbench/embedding/methods/numeric_features/metadata.json +29 -0
  778. ragbench/embedding/methods/numeric_features/requirements.txt +1 -0
  779. ragbench/embedding/methods/ollama/__init__.py +3 -0
  780. ragbench/embedding/methods/ollama/embedder.py +20 -0
  781. ragbench/embedding/methods/ollama/metadata.json +35 -0
  782. ragbench/embedding/methods/ollama/requirements.txt +0 -0
  783. ragbench/embedding/methods/openai/__init__.py +3 -0
  784. ragbench/embedding/methods/openai/embedder.py +10 -0
  785. ragbench/embedding/methods/openai/metadata.json +36 -0
  786. ragbench/embedding/methods/openai/requirements.txt +0 -0
  787. ragbench/embedding/methods/openai_compatible/__init__.py +3 -0
  788. ragbench/embedding/methods/openai_compatible/embedder.py +10 -0
  789. ragbench/embedding/methods/openai_compatible/metadata.json +40 -0
  790. ragbench/embedding/methods/openai_compatible/requirements.txt +0 -0
  791. ragbench/embedding/methods/pca_reduce/__init__.py +3 -0
  792. ragbench/embedding/methods/pca_reduce/embedder.py +32 -0
  793. ragbench/embedding/methods/pca_reduce/metadata.json +33 -0
  794. ragbench/embedding/methods/pca_reduce/requirements.txt +0 -0
  795. ragbench/embedding/methods/quantize_binary/__init__.py +3 -0
  796. ragbench/embedding/methods/quantize_binary/embedder.py +17 -0
  797. ragbench/embedding/methods/quantize_binary/metadata.json +27 -0
  798. ragbench/embedding/methods/quantize_binary/requirements.txt +0 -0
  799. ragbench/embedding/methods/quantize_int8/__init__.py +3 -0
  800. ragbench/embedding/methods/quantize_int8/embedder.py +15 -0
  801. ragbench/embedding/methods/quantize_int8/metadata.json +26 -0
  802. ragbench/embedding/methods/quantize_int8/requirements.txt +0 -0
  803. ragbench/embedding/methods/sentence_transformers/__init__.py +1 -0
  804. ragbench/embedding/methods/sentence_transformers/embedder.py +96 -0
  805. ragbench/embedding/methods/sentence_transformers/metadata.json +20 -0
  806. ragbench/embedding/methods/sentence_transformers/requirements.txt +2 -0
  807. ragbench/embedding/methods/siglip/__init__.py +3 -0
  808. ragbench/embedding/methods/siglip/embedder.py +39 -0
  809. ragbench/embedding/methods/siglip/metadata.json +29 -0
  810. ragbench/embedding/methods/siglip/requirements.txt +3 -0
  811. ragbench/embedding/methods/splade/__init__.py +3 -0
  812. ragbench/embedding/methods/splade/embedder.py +38 -0
  813. ragbench/embedding/methods/splade/metadata.json +32 -0
  814. ragbench/embedding/methods/splade/requirements.txt +2 -0
  815. ragbench/embedding/methods/tabular_text/__init__.py +3 -0
  816. ragbench/embedding/methods/tabular_text/embedder.py +56 -0
  817. ragbench/embedding/methods/tabular_text/metadata.json +30 -0
  818. ragbench/embedding/methods/tabular_text/requirements.txt +0 -0
  819. ragbench/embedding/methods/tei/__init__.py +3 -0
  820. ragbench/embedding/methods/tei/embedder.py +18 -0
  821. ragbench/embedding/methods/tei/metadata.json +39 -0
  822. ragbench/embedding/methods/tei/requirements.txt +0 -0
  823. ragbench/embedding/methods/tfidf/__init__.py +3 -0
  824. ragbench/embedding/methods/tfidf/embedder.py +41 -0
  825. ragbench/embedding/methods/tfidf/metadata.json +23 -0
  826. ragbench/embedding/methods/tfidf/requirements.txt +0 -0
  827. ragbench/embedding/methods/transe/__init__.py +3 -0
  828. ragbench/embedding/methods/transe/embedder.py +111 -0
  829. ragbench/embedding/methods/transe/metadata.json +39 -0
  830. ragbench/embedding/methods/transe/requirements.txt +1 -0
  831. ragbench/embedding/methods/video_frames/__init__.py +3 -0
  832. ragbench/embedding/methods/video_frames/embedder.py +62 -0
  833. ragbench/embedding/methods/video_frames/metadata.json +30 -0
  834. ragbench/embedding/methods/video_frames/requirements.txt +1 -0
  835. ragbench/embedding/methods/vllm/__init__.py +3 -0
  836. ragbench/embedding/methods/vllm/embedder.py +10 -0
  837. ragbench/embedding/methods/vllm/metadata.json +37 -0
  838. ragbench/embedding/methods/vllm/requirements.txt +0 -0
  839. ragbench/embedding/methods/voyage/__init__.py +3 -0
  840. ragbench/embedding/methods/voyage/embedder.py +13 -0
  841. ragbench/embedding/methods/voyage/metadata.json +35 -0
  842. ragbench/embedding/methods/voyage/requirements.txt +0 -0
  843. ragbench/embedding/methods/word2vec/__init__.py +3 -0
  844. ragbench/embedding/methods/word2vec/embedder.py +70 -0
  845. ragbench/embedding/methods/word2vec/metadata.json +23 -0
  846. ragbench/embedding/methods/word2vec/requirements.txt +0 -0
  847. ragbench/embedding/models.py +84 -0
  848. ragbench/embedding/registry.py +139 -0
  849. ragbench/evaluation/BENCHMARKING.md +538 -0
  850. ragbench/evaluation/SWEEP_README.md +398 -0
  851. ragbench/evaluation/__init__.py +70 -0
  852. ragbench/evaluation/attribution.py +177 -0
  853. ragbench/evaluation/benchmark.py +476 -0
  854. ragbench/evaluation/ci_gate.py +164 -0
  855. ragbench/evaluation/evaluator.py +228 -0
  856. ragbench/evaluation/metrics/__init__.py +1 -0
  857. ragbench/evaluation/metrics/abstention/metadata.json +7 -0
  858. ragbench/evaluation/metrics/abstention/metric.py +110 -0
  859. ragbench/evaluation/metrics/answer_correctness/metadata.json +7 -0
  860. ragbench/evaluation/metrics/answer_correctness/metric.py +120 -0
  861. ragbench/evaluation/metrics/answer_relevancy/metadata.json +7 -0
  862. ragbench/evaluation/metrics/answer_relevancy/metric.py +116 -0
  863. ragbench/evaluation/metrics/base.py +138 -0
  864. ragbench/evaluation/metrics/citation_quality/metadata.json +7 -0
  865. ragbench/evaluation/metrics/citation_quality/metric.py +110 -0
  866. ragbench/evaluation/metrics/context_relevance/metadata.json +7 -0
  867. ragbench/evaluation/metrics/context_relevance/metric.py +122 -0
  868. ragbench/evaluation/metrics/faithfulness/metadata.json +7 -0
  869. ragbench/evaluation/metrics/faithfulness/metric.py +204 -0
  870. ragbench/evaluation/metrics/retrieval_ranking/metadata.json +7 -0
  871. ragbench/evaluation/metrics/retrieval_ranking/metric.py +165 -0
  872. ragbench/evaluation/metrics/system_metrics/metadata.json +7 -0
  873. ragbench/evaluation/metrics/system_metrics/metric.py +131 -0
  874. ragbench/evaluation/metrics/text_overlap/metadata.json +7 -0
  875. ragbench/evaluation/metrics/text_overlap/metric.py +311 -0
  876. ragbench/evaluation/models.py +238 -0
  877. ragbench/evaluation/registry.py +201 -0
  878. ragbench/evaluation/reporters.py +301 -0
  879. ragbench/evaluation/sweep.py +340 -0
  880. ragbench/evaluation/testset_generator.py +445 -0
  881. ragbench/generation/PROVIDERS.md +665 -0
  882. ragbench/generation/PROVIDERS_QUICK_REFERENCE.md +296 -0
  883. ragbench/generation/README.md +890 -0
  884. ragbench/generation/__init__.py +85 -0
  885. ragbench/generation/base.py +244 -0
  886. ragbench/generation/context_adapter.py +176 -0
  887. ragbench/generation/current_models.json +180 -0
  888. ragbench/generation/env.py +203 -0
  889. ragbench/generation/errors.py +49 -0
  890. ragbench/generation/generator.py +151 -0
  891. ragbench/generation/methods/anthropic/__init__.py +4 -0
  892. ragbench/generation/methods/anthropic/generator.py +288 -0
  893. ragbench/generation/methods/cohere/__init__.py +4 -0
  894. ragbench/generation/methods/cohere/generator.py +290 -0
  895. ragbench/generation/methods/gemini/__init__.py +4 -0
  896. ragbench/generation/methods/gemini/generator.py +284 -0
  897. ragbench/generation/methods/groq/__init__.py +4 -0
  898. ragbench/generation/methods/groq/generator.py +334 -0
  899. ragbench/generation/methods/mistral/__init__.py +4 -0
  900. ragbench/generation/methods/mistral/generator.py +333 -0
  901. ragbench/generation/methods/ollama/__init__.py +4 -0
  902. ragbench/generation/methods/ollama/generator.py +258 -0
  903. ragbench/generation/methods/openai/__init__.py +4 -0
  904. ragbench/generation/methods/openai/generator.py +302 -0
  905. ragbench/generation/methods/openrouter/__init__.py +4 -0
  906. ragbench/generation/methods/openrouter/generator.py +351 -0
  907. ragbench/generation/methods/together/__init__.py +4 -0
  908. ragbench/generation/methods/together/generator.py +334 -0
  909. ragbench/generation/model_config.py +209 -0
  910. ragbench/generation/model_validator.py +122 -0
  911. ragbench/generation/models.py +159 -0
  912. ragbench/generation/registry.py +101 -0
  913. ragbench/generation/requirements.txt +14 -0
  914. ragbench/ingestion/__init__.py +28 -0
  915. ragbench/pipeline/__init__.py +20 -0
  916. ragbench/pipeline/embedding_routing.py +88 -0
  917. ragbench/pipeline/pipeline.py +676 -0
  918. ragbench/pipeline/retrieval_routing.py +45 -0
  919. ragbench/pipeline/routing.py +70 -0
  920. ragbench/pipeline/vectorstore_routing.py +92 -0
  921. ragbench/reranking/__init__.py +36 -0
  922. ragbench/reranking/compression.py +61 -0
  923. ragbench/reranking/dedup.py +134 -0
  924. ragbench/reranking/env.py +80 -0
  925. ragbench/reranking/errors.py +43 -0
  926. ragbench/reranking/methods/__init__.py +4 -0
  927. ragbench/reranking/methods/_helpers.py +339 -0
  928. ragbench/reranking/methods/_http.py +180 -0
  929. ragbench/reranking/methods/auto/__init__.py +1 -0
  930. ragbench/reranking/methods/auto/metadata.json +14 -0
  931. ragbench/reranking/methods/auto/requirements.txt +1 -0
  932. ragbench/reranking/methods/auto/reranker.py +6 -0
  933. ragbench/reranking/methods/base.py +59 -0
  934. ragbench/reranking/methods/bge_reranker/__init__.py +0 -0
  935. ragbench/reranking/methods/bge_reranker/metadata.json +17 -0
  936. ragbench/reranking/methods/bge_reranker/requirements.txt +1 -0
  937. ragbench/reranking/methods/bge_reranker/reranker.py +5 -0
  938. ragbench/reranking/methods/bm25/__init__.py +1 -0
  939. ragbench/reranking/methods/bm25/metadata.json +14 -0
  940. ragbench/reranking/methods/bm25/requirements.txt +1 -0
  941. ragbench/reranking/methods/bm25/reranker.py +6 -0
  942. ragbench/reranking/methods/cohere/__init__.py +0 -0
  943. ragbench/reranking/methods/cohere/metadata.json +18 -0
  944. ragbench/reranking/methods/cohere/requirements.txt +0 -0
  945. ragbench/reranking/methods/cohere/reranker.py +4 -0
  946. ragbench/reranking/methods/colbert/__init__.py +0 -0
  947. ragbench/reranking/methods/colbert/metadata.json +15 -0
  948. ragbench/reranking/methods/colbert/requirements.txt +0 -0
  949. ragbench/reranking/methods/colbert/reranker.py +5 -0
  950. ragbench/reranking/methods/compression/__init__.py +1 -0
  951. ragbench/reranking/methods/compression/metadata.json +14 -0
  952. ragbench/reranking/methods/compression/requirements.txt +1 -0
  953. ragbench/reranking/methods/compression/reranker.py +6 -0
  954. ragbench/reranking/methods/custom_callable/__init__.py +1 -0
  955. ragbench/reranking/methods/custom_callable/metadata.json +14 -0
  956. ragbench/reranking/methods/custom_callable/requirements.txt +1 -0
  957. ragbench/reranking/methods/custom_callable/reranker.py +6 -0
  958. ragbench/reranking/methods/dedup/__init__.py +1 -0
  959. ragbench/reranking/methods/dedup/metadata.json +14 -0
  960. ragbench/reranking/methods/dedup/requirements.txt +1 -0
  961. ragbench/reranking/methods/dedup/reranker.py +6 -0
  962. ragbench/reranking/methods/diversity/__init__.py +1 -0
  963. ragbench/reranking/methods/diversity/metadata.json +14 -0
  964. ragbench/reranking/methods/diversity/requirements.txt +1 -0
  965. ragbench/reranking/methods/diversity/reranker.py +6 -0
  966. ragbench/reranking/methods/fusion/__init__.py +1 -0
  967. ragbench/reranking/methods/fusion/metadata.json +14 -0
  968. ragbench/reranking/methods/fusion/requirements.txt +1 -0
  969. ragbench/reranking/methods/fusion/reranker.py +6 -0
  970. ragbench/reranking/methods/jina/__init__.py +0 -0
  971. ragbench/reranking/methods/jina/metadata.json +18 -0
  972. ragbench/reranking/methods/jina/requirements.txt +0 -0
  973. ragbench/reranking/methods/jina/reranker.py +4 -0
  974. ragbench/reranking/methods/lexical/__init__.py +1 -0
  975. ragbench/reranking/methods/lexical/metadata.json +14 -0
  976. ragbench/reranking/methods/lexical/requirements.txt +1 -0
  977. ragbench/reranking/methods/lexical/reranker.py +6 -0
  978. ragbench/reranking/methods/llm_listwise/__init__.py +1 -0
  979. ragbench/reranking/methods/llm_listwise/metadata.json +14 -0
  980. ragbench/reranking/methods/llm_listwise/requirements.txt +1 -0
  981. ragbench/reranking/methods/llm_listwise/reranker.py +6 -0
  982. ragbench/reranking/methods/llm_pairwise/__init__.py +1 -0
  983. ragbench/reranking/methods/llm_pairwise/metadata.json +14 -0
  984. ragbench/reranking/methods/llm_pairwise/requirements.txt +1 -0
  985. ragbench/reranking/methods/llm_pairwise/reranker.py +6 -0
  986. ragbench/reranking/methods/metadata_boost/__init__.py +1 -0
  987. ragbench/reranking/methods/metadata_boost/metadata.json +14 -0
  988. ragbench/reranking/methods/metadata_boost/requirements.txt +1 -0
  989. ragbench/reranking/methods/metadata_boost/reranker.py +6 -0
  990. ragbench/reranking/methods/minilm/__init__.py +0 -0
  991. ragbench/reranking/methods/minilm/metadata.json +17 -0
  992. ragbench/reranking/methods/minilm/requirements.txt +1 -0
  993. ragbench/reranking/methods/minilm/reranker.py +5 -0
  994. ragbench/reranking/methods/noop/__init__.py +1 -0
  995. ragbench/reranking/methods/noop/metadata.json +14 -0
  996. ragbench/reranking/methods/noop/requirements.txt +1 -0
  997. ragbench/reranking/methods/noop/reranker.py +6 -0
  998. ragbench/reranking/methods/normalize/__init__.py +1 -0
  999. ragbench/reranking/methods/normalize/metadata.json +14 -0
  1000. ragbench/reranking/methods/normalize/requirements.txt +1 -0
  1001. ragbench/reranking/methods/normalize/reranker.py +6 -0
  1002. ragbench/reranking/methods/threshold/__init__.py +1 -0
  1003. ragbench/reranking/methods/threshold/metadata.json +14 -0
  1004. ragbench/reranking/methods/threshold/requirements.txt +1 -0
  1005. ragbench/reranking/methods/threshold/reranker.py +6 -0
  1006. ragbench/reranking/methods/voyage/__init__.py +0 -0
  1007. ragbench/reranking/methods/voyage/metadata.json +18 -0
  1008. ragbench/reranking/methods/voyage/requirements.txt +0 -0
  1009. ragbench/reranking/methods/voyage/reranker.py +4 -0
  1010. ragbench/reranking/methods/window_compression/__init__.py +1 -0
  1011. ragbench/reranking/methods/window_compression/metadata.json +14 -0
  1012. ragbench/reranking/methods/window_compression/requirements.txt +1 -0
  1013. ragbench/reranking/methods/window_compression/reranker.py +6 -0
  1014. ragbench/reranking/models.py +157 -0
  1015. ragbench/reranking/registry.py +161 -0
  1016. ragbench/reranking/reranker.py +289 -0
  1017. ragbench/reranking/routing.py +158 -0
  1018. ragbench/reranking/text.py +124 -0
  1019. ragbench/retrieval/README.md +83 -0
  1020. ragbench/retrieval/__init__.py +30 -0
  1021. ragbench/retrieval/diversity.py +111 -0
  1022. ragbench/retrieval/env.py +80 -0
  1023. ragbench/retrieval/errors.py +55 -0
  1024. ragbench/retrieval/expansion.py +149 -0
  1025. ragbench/retrieval/filters.py +107 -0
  1026. ragbench/retrieval/fusion.py +152 -0
  1027. ragbench/retrieval/methods/__init__.py +1 -0
  1028. ragbench/retrieval/methods/_helpers.py +309 -0
  1029. ragbench/retrieval/methods/auto/__init__.py +1 -0
  1030. ragbench/retrieval/methods/auto/metadata.json +27 -0
  1031. ragbench/retrieval/methods/auto/requirements.txt +0 -0
  1032. ragbench/retrieval/methods/auto/retriever.py +6 -0
  1033. ragbench/retrieval/methods/base.py +69 -0
  1034. ragbench/retrieval/methods/bm25/__init__.py +1 -0
  1035. ragbench/retrieval/methods/bm25/metadata.json +23 -0
  1036. ragbench/retrieval/methods/bm25/requirements.txt +0 -0
  1037. ragbench/retrieval/methods/bm25/retriever.py +6 -0
  1038. ragbench/retrieval/methods/custom_callable/__init__.py +1 -0
  1039. ragbench/retrieval/methods/custom_callable/metadata.json +24 -0
  1040. ragbench/retrieval/methods/custom_callable/requirements.txt +0 -0
  1041. ragbench/retrieval/methods/custom_callable/retriever.py +6 -0
  1042. ragbench/retrieval/methods/decomposition/__init__.py +1 -0
  1043. ragbench/retrieval/methods/decomposition/metadata.json +28 -0
  1044. ragbench/retrieval/methods/decomposition/requirements.txt +0 -0
  1045. ragbench/retrieval/methods/decomposition/retriever.py +6 -0
  1046. ragbench/retrieval/methods/dense/__init__.py +1 -0
  1047. ragbench/retrieval/methods/dense/metadata.json +24 -0
  1048. ragbench/retrieval/methods/dense/requirements.txt +0 -0
  1049. ragbench/retrieval/methods/dense/retriever.py +6 -0
  1050. ragbench/retrieval/methods/dense_filtered/__init__.py +1 -0
  1051. ragbench/retrieval/methods/dense_filtered/metadata.json +24 -0
  1052. ragbench/retrieval/methods/dense_filtered/requirements.txt +0 -0
  1053. ragbench/retrieval/methods/dense_filtered/retriever.py +6 -0
  1054. ragbench/retrieval/methods/diversity_rerank/__init__.py +1 -0
  1055. ragbench/retrieval/methods/diversity_rerank/metadata.json +26 -0
  1056. ragbench/retrieval/methods/diversity_rerank/requirements.txt +0 -0
  1057. ragbench/retrieval/methods/diversity_rerank/retriever.py +6 -0
  1058. ragbench/retrieval/methods/filtered/__init__.py +1 -0
  1059. ragbench/retrieval/methods/filtered/metadata.json +30 -0
  1060. ragbench/retrieval/methods/filtered/requirements.txt +0 -0
  1061. ragbench/retrieval/methods/filtered/retriever.py +6 -0
  1062. ragbench/retrieval/methods/hash_baseline/__init__.py +1 -0
  1063. ragbench/retrieval/methods/hash_baseline/metadata.json +22 -0
  1064. ragbench/retrieval/methods/hash_baseline/requirements.txt +0 -0
  1065. ragbench/retrieval/methods/hash_baseline/retriever.py +6 -0
  1066. ragbench/retrieval/methods/hybrid_rerank/__init__.py +1 -0
  1067. ragbench/retrieval/methods/hybrid_rerank/metadata.json +24 -0
  1068. ragbench/retrieval/methods/hybrid_rerank/requirements.txt +0 -0
  1069. ragbench/retrieval/methods/hybrid_rerank/retriever.py +6 -0
  1070. ragbench/retrieval/methods/hybrid_rrf/__init__.py +1 -0
  1071. ragbench/retrieval/methods/hybrid_rrf/metadata.json +29 -0
  1072. ragbench/retrieval/methods/hybrid_rrf/requirements.txt +0 -0
  1073. ragbench/retrieval/methods/hybrid_rrf/retriever.py +6 -0
  1074. ragbench/retrieval/methods/hybrid_weighted/__init__.py +1 -0
  1075. ragbench/retrieval/methods/hybrid_weighted/metadata.json +28 -0
  1076. ragbench/retrieval/methods/hybrid_weighted/requirements.txt +0 -0
  1077. ragbench/retrieval/methods/hybrid_weighted/retriever.py +6 -0
  1078. ragbench/retrieval/methods/hyde/__init__.py +1 -0
  1079. ragbench/retrieval/methods/hyde/metadata.json +26 -0
  1080. ragbench/retrieval/methods/hyde/requirements.txt +0 -0
  1081. ragbench/retrieval/methods/hyde/retriever.py +6 -0
  1082. ragbench/retrieval/methods/mmr/__init__.py +1 -0
  1083. ragbench/retrieval/methods/mmr/metadata.json +26 -0
  1084. ragbench/retrieval/methods/mmr/requirements.txt +0 -0
  1085. ragbench/retrieval/methods/mmr/retriever.py +6 -0
  1086. ragbench/retrieval/methods/multi_collection/__init__.py +1 -0
  1087. ragbench/retrieval/methods/multi_collection/metadata.json +27 -0
  1088. ragbench/retrieval/methods/multi_collection/requirements.txt +0 -0
  1089. ragbench/retrieval/methods/multi_collection/retriever.py +6 -0
  1090. ragbench/retrieval/methods/multi_collection_rrf/__init__.py +1 -0
  1091. ragbench/retrieval/methods/multi_collection_rrf/metadata.json +27 -0
  1092. ragbench/retrieval/methods/multi_collection_rrf/requirements.txt +0 -0
  1093. ragbench/retrieval/methods/multi_collection_rrf/retriever.py +6 -0
  1094. ragbench/retrieval/methods/multi_query/__init__.py +1 -0
  1095. ragbench/retrieval/methods/multi_query/metadata.json +30 -0
  1096. ragbench/retrieval/methods/multi_query/requirements.txt +0 -0
  1097. ragbench/retrieval/methods/multi_query/retriever.py +6 -0
  1098. ragbench/retrieval/methods/multi_vector/__init__.py +1 -0
  1099. ragbench/retrieval/methods/multi_vector/metadata.json +24 -0
  1100. ragbench/retrieval/methods/multi_vector/requirements.txt +0 -0
  1101. ragbench/retrieval/methods/multi_vector/retriever.py +6 -0
  1102. ragbench/retrieval/methods/parent_child/__init__.py +1 -0
  1103. ragbench/retrieval/methods/parent_child/metadata.json +30 -0
  1104. ragbench/retrieval/methods/parent_child/requirements.txt +0 -0
  1105. ragbench/retrieval/methods/parent_child/retriever.py +6 -0
  1106. ragbench/retrieval/methods/query_rewrite/__init__.py +1 -0
  1107. ragbench/retrieval/methods/query_rewrite/metadata.json +30 -0
  1108. ragbench/retrieval/methods/query_rewrite/requirements.txt +0 -0
  1109. ragbench/retrieval/methods/query_rewrite/retriever.py +6 -0
  1110. ragbench/retrieval/methods/routed/__init__.py +1 -0
  1111. ragbench/retrieval/methods/routed/metadata.json +27 -0
  1112. ragbench/retrieval/methods/routed/requirements.txt +0 -0
  1113. ragbench/retrieval/methods/routed/retriever.py +6 -0
  1114. ragbench/retrieval/methods/rrf_fusion/__init__.py +1 -0
  1115. ragbench/retrieval/methods/rrf_fusion/metadata.json +29 -0
  1116. ragbench/retrieval/methods/rrf_fusion/requirements.txt +0 -0
  1117. ragbench/retrieval/methods/rrf_fusion/retriever.py +6 -0
  1118. ragbench/retrieval/methods/self_query/__init__.py +1 -0
  1119. ragbench/retrieval/methods/self_query/metadata.json +26 -0
  1120. ragbench/retrieval/methods/self_query/requirements.txt +0 -0
  1121. ragbench/retrieval/methods/self_query/retriever.py +6 -0
  1122. ragbench/retrieval/methods/sentence_window/__init__.py +1 -0
  1123. ragbench/retrieval/methods/sentence_window/metadata.json +33 -0
  1124. ragbench/retrieval/methods/sentence_window/requirements.txt +0 -0
  1125. ragbench/retrieval/methods/sentence_window/retriever.py +6 -0
  1126. ragbench/retrieval/methods/similarity_threshold/__init__.py +1 -0
  1127. ragbench/retrieval/methods/similarity_threshold/metadata.json +30 -0
  1128. ragbench/retrieval/methods/similarity_threshold/requirements.txt +0 -0
  1129. ragbench/retrieval/methods/similarity_threshold/retriever.py +6 -0
  1130. ragbench/retrieval/methods/sparse/__init__.py +1 -0
  1131. ragbench/retrieval/methods/sparse/metadata.json +23 -0
  1132. ragbench/retrieval/methods/sparse/requirements.txt +0 -0
  1133. ragbench/retrieval/methods/sparse/retriever.py +6 -0
  1134. ragbench/retrieval/methods/step_back/__init__.py +1 -0
  1135. ragbench/retrieval/methods/step_back/metadata.json +26 -0
  1136. ragbench/retrieval/methods/step_back/requirements.txt +0 -0
  1137. ragbench/retrieval/methods/step_back/retriever.py +6 -0
  1138. ragbench/retrieval/methods/weighted_fusion/__init__.py +1 -0
  1139. ragbench/retrieval/methods/weighted_fusion/metadata.json +26 -0
  1140. ragbench/retrieval/methods/weighted_fusion/requirements.txt +0 -0
  1141. ragbench/retrieval/methods/weighted_fusion/retriever.py +6 -0
  1142. ragbench/retrieval/models.py +168 -0
  1143. ragbench/retrieval/query.py +160 -0
  1144. ragbench/retrieval/registry.py +123 -0
  1145. ragbench/retrieval/retriever.py +347 -0
  1146. ragbench/retrieval/routing.py +130 -0
  1147. ragbench/retrieval/self_query.py +62 -0
  1148. ragbench/vectorstore/README.md +374 -0
  1149. ragbench/vectorstore/__init__.py +32 -0
  1150. ragbench/vectorstore/env.py +78 -0
  1151. ragbench/vectorstore/errors.py +59 -0
  1152. ragbench/vectorstore/filters.py +167 -0
  1153. ragbench/vectorstore/methods/__init__.py +4 -0
  1154. ragbench/vectorstore/methods/_ann.py +99 -0
  1155. ragbench/vectorstore/methods/_bruteforce.py +170 -0
  1156. ragbench/vectorstore/methods/_elastic.py +166 -0
  1157. ragbench/vectorstore/methods/_embedded.py +49 -0
  1158. ragbench/vectorstore/methods/_http.py +141 -0
  1159. ragbench/vectorstore/methods/_qdrant_common.py +53 -0
  1160. ragbench/vectorstore/methods/_scan.py +57 -0
  1161. ragbench/vectorstore/methods/_sql.py +205 -0
  1162. ragbench/vectorstore/methods/_wrapper.py +66 -0
  1163. ragbench/vectorstore/methods/annoy/__init__.py +3 -0
  1164. ragbench/vectorstore/methods/annoy/metadata.json +39 -0
  1165. ragbench/vectorstore/methods/annoy/requirements.txt +2 -0
  1166. ragbench/vectorstore/methods/annoy/store.py +25 -0
  1167. ragbench/vectorstore/methods/astra_db/__init__.py +3 -0
  1168. ragbench/vectorstore/methods/astra_db/metadata.json +61 -0
  1169. ragbench/vectorstore/methods/astra_db/requirements.txt +0 -0
  1170. ragbench/vectorstore/methods/astra_db/store.py +100 -0
  1171. ragbench/vectorstore/methods/azure_ai_search/__init__.py +3 -0
  1172. ragbench/vectorstore/methods/azure_ai_search/metadata.json +59 -0
  1173. ragbench/vectorstore/methods/azure_ai_search/requirements.txt +0 -0
  1174. ragbench/vectorstore/methods/azure_ai_search/store.py +83 -0
  1175. ragbench/vectorstore/methods/base.py +607 -0
  1176. ragbench/vectorstore/methods/bm25_local/__init__.py +3 -0
  1177. ragbench/vectorstore/methods/bm25_local/metadata.json +43 -0
  1178. ragbench/vectorstore/methods/bm25_local/requirements.txt +0 -0
  1179. ragbench/vectorstore/methods/bm25_local/store.py +133 -0
  1180. ragbench/vectorstore/methods/chroma/__init__.py +3 -0
  1181. ragbench/vectorstore/methods/chroma/metadata.json +40 -0
  1182. ragbench/vectorstore/methods/chroma/requirements.txt +1 -0
  1183. ragbench/vectorstore/methods/chroma/store.py +120 -0
  1184. ragbench/vectorstore/methods/clickhouse/__init__.py +3 -0
  1185. ragbench/vectorstore/methods/clickhouse/metadata.json +64 -0
  1186. ragbench/vectorstore/methods/clickhouse/requirements.txt +1 -0
  1187. ragbench/vectorstore/methods/clickhouse/store.py +142 -0
  1188. ragbench/vectorstore/methods/custom_callable/__init__.py +3 -0
  1189. ragbench/vectorstore/methods/custom_callable/metadata.json +75 -0
  1190. ragbench/vectorstore/methods/custom_callable/requirements.txt +0 -0
  1191. ragbench/vectorstore/methods/custom_callable/store.py +40 -0
  1192. ragbench/vectorstore/methods/dedup/__init__.py +3 -0
  1193. ragbench/vectorstore/methods/dedup/metadata.json +54 -0
  1194. ragbench/vectorstore/methods/dedup/requirements.txt +0 -0
  1195. ragbench/vectorstore/methods/dedup/store.py +38 -0
  1196. ragbench/vectorstore/methods/duckdb/__init__.py +3 -0
  1197. ragbench/vectorstore/methods/duckdb/metadata.json +40 -0
  1198. ragbench/vectorstore/methods/duckdb/requirements.txt +1 -0
  1199. ragbench/vectorstore/methods/duckdb/store.py +116 -0
  1200. ragbench/vectorstore/methods/elasticsearch/__init__.py +3 -0
  1201. ragbench/vectorstore/methods/elasticsearch/metadata.json +58 -0
  1202. ragbench/vectorstore/methods/elasticsearch/requirements.txt +0 -0
  1203. ragbench/vectorstore/methods/elasticsearch/store.py +23 -0
  1204. ragbench/vectorstore/methods/faiss/__init__.py +3 -0
  1205. ragbench/vectorstore/methods/faiss/metadata.json +45 -0
  1206. ragbench/vectorstore/methods/faiss/requirements.txt +2 -0
  1207. ragbench/vectorstore/methods/faiss/store.py +52 -0
  1208. ragbench/vectorstore/methods/flat_file/__init__.py +3 -0
  1209. ragbench/vectorstore/methods/flat_file/metadata.json +40 -0
  1210. ragbench/vectorstore/methods/flat_file/requirements.txt +0 -0
  1211. ragbench/vectorstore/methods/flat_file/store.py +108 -0
  1212. ragbench/vectorstore/methods/hnswlib/__init__.py +3 -0
  1213. ragbench/vectorstore/methods/hnswlib/metadata.json +39 -0
  1214. ragbench/vectorstore/methods/hnswlib/requirements.txt +2 -0
  1215. ragbench/vectorstore/methods/hnswlib/store.py +28 -0
  1216. ragbench/vectorstore/methods/lancedb/__init__.py +3 -0
  1217. ragbench/vectorstore/methods/lancedb/metadata.json +41 -0
  1218. ragbench/vectorstore/methods/lancedb/requirements.txt +2 -0
  1219. ragbench/vectorstore/methods/lancedb/store.py +115 -0
  1220. ragbench/vectorstore/methods/mariadb/__init__.py +3 -0
  1221. ragbench/vectorstore/methods/mariadb/metadata.json +64 -0
  1222. ragbench/vectorstore/methods/mariadb/requirements.txt +1 -0
  1223. ragbench/vectorstore/methods/mariadb/store.py +39 -0
  1224. ragbench/vectorstore/methods/meilisearch/__init__.py +3 -0
  1225. ragbench/vectorstore/methods/meilisearch/metadata.json +55 -0
  1226. ragbench/vectorstore/methods/meilisearch/requirements.txt +0 -0
  1227. ragbench/vectorstore/methods/meilisearch/store.py +83 -0
  1228. ragbench/vectorstore/methods/memory/__init__.py +3 -0
  1229. ragbench/vectorstore/methods/memory/metadata.json +33 -0
  1230. ragbench/vectorstore/methods/memory/requirements.txt +0 -0
  1231. ragbench/vectorstore/methods/memory/store.py +5 -0
  1232. ragbench/vectorstore/methods/milvus/__init__.py +3 -0
  1233. ragbench/vectorstore/methods/milvus/metadata.json +60 -0
  1234. ragbench/vectorstore/methods/milvus/requirements.txt +0 -0
  1235. ragbench/vectorstore/methods/milvus/store.py +91 -0
  1236. ragbench/vectorstore/methods/milvus_lite/__init__.py +3 -0
  1237. ragbench/vectorstore/methods/milvus_lite/metadata.json +42 -0
  1238. ragbench/vectorstore/methods/milvus_lite/requirements.txt +2 -0
  1239. ragbench/vectorstore/methods/milvus_lite/store.py +124 -0
  1240. ragbench/vectorstore/methods/mirror/__init__.py +3 -0
  1241. ragbench/vectorstore/methods/mirror/metadata.json +59 -0
  1242. ragbench/vectorstore/methods/mirror/requirements.txt +0 -0
  1243. ragbench/vectorstore/methods/mirror/store.py +49 -0
  1244. ragbench/vectorstore/methods/mongodb_atlas/__init__.py +3 -0
  1245. ragbench/vectorstore/methods/mongodb_atlas/metadata.json +59 -0
  1246. ragbench/vectorstore/methods/mongodb_atlas/requirements.txt +1 -0
  1247. ragbench/vectorstore/methods/mongodb_atlas/store.py +92 -0
  1248. ragbench/vectorstore/methods/namespace/__init__.py +3 -0
  1249. ragbench/vectorstore/methods/namespace/metadata.json +58 -0
  1250. ragbench/vectorstore/methods/namespace/requirements.txt +0 -0
  1251. ragbench/vectorstore/methods/namespace/store.py +51 -0
  1252. ragbench/vectorstore/methods/neo4j/__init__.py +3 -0
  1253. ragbench/vectorstore/methods/neo4j/metadata.json +64 -0
  1254. ragbench/vectorstore/methods/neo4j/requirements.txt +0 -0
  1255. ragbench/vectorstore/methods/neo4j/store.py +97 -0
  1256. ragbench/vectorstore/methods/numpy_flat/__init__.py +3 -0
  1257. ragbench/vectorstore/methods/numpy_flat/metadata.json +40 -0
  1258. ragbench/vectorstore/methods/numpy_flat/requirements.txt +1 -0
  1259. ragbench/vectorstore/methods/numpy_flat/store.py +113 -0
  1260. ragbench/vectorstore/methods/opensearch/__init__.py +3 -0
  1261. ragbench/vectorstore/methods/opensearch/metadata.json +64 -0
  1262. ragbench/vectorstore/methods/opensearch/requirements.txt +0 -0
  1263. ragbench/vectorstore/methods/opensearch/store.py +40 -0
  1264. ragbench/vectorstore/methods/oracle_23ai/__init__.py +3 -0
  1265. ragbench/vectorstore/methods/oracle_23ai/metadata.json +62 -0
  1266. ragbench/vectorstore/methods/oracle_23ai/requirements.txt +1 -0
  1267. ragbench/vectorstore/methods/oracle_23ai/store.py +48 -0
  1268. ragbench/vectorstore/methods/pgvector/__init__.py +3 -0
  1269. ragbench/vectorstore/methods/pgvector/metadata.json +58 -0
  1270. ragbench/vectorstore/methods/pgvector/requirements.txt +1 -0
  1271. ragbench/vectorstore/methods/pgvector/store.py +133 -0
  1272. ragbench/vectorstore/methods/pinecone/__init__.py +3 -0
  1273. ragbench/vectorstore/methods/pinecone/metadata.json +59 -0
  1274. ragbench/vectorstore/methods/pinecone/requirements.txt +0 -0
  1275. ragbench/vectorstore/methods/pinecone/store.py +148 -0
  1276. ragbench/vectorstore/methods/qdrant/__init__.py +3 -0
  1277. ragbench/vectorstore/methods/qdrant/metadata.json +61 -0
  1278. ragbench/vectorstore/methods/qdrant/requirements.txt +0 -0
  1279. ragbench/vectorstore/methods/qdrant/store.py +127 -0
  1280. ragbench/vectorstore/methods/qdrant_local/__init__.py +3 -0
  1281. ragbench/vectorstore/methods/qdrant_local/metadata.json +58 -0
  1282. ragbench/vectorstore/methods/qdrant_local/requirements.txt +1 -0
  1283. ragbench/vectorstore/methods/qdrant_local/store.py +147 -0
  1284. ragbench/vectorstore/methods/read_cache/__init__.py +3 -0
  1285. ragbench/vectorstore/methods/read_cache/metadata.json +57 -0
  1286. ragbench/vectorstore/methods/read_cache/requirements.txt +0 -0
  1287. ragbench/vectorstore/methods/read_cache/store.py +36 -0
  1288. ragbench/vectorstore/methods/redis/__init__.py +3 -0
  1289. ragbench/vectorstore/methods/redis/metadata.json +57 -0
  1290. ragbench/vectorstore/methods/redis/requirements.txt +1 -0
  1291. ragbench/vectorstore/methods/redis/store.py +156 -0
  1292. ragbench/vectorstore/methods/shard_router/__init__.py +3 -0
  1293. ragbench/vectorstore/methods/shard_router/metadata.json +56 -0
  1294. ragbench/vectorstore/methods/shard_router/requirements.txt +0 -0
  1295. ragbench/vectorstore/methods/shard_router/store.py +101 -0
  1296. ragbench/vectorstore/methods/singlestore/__init__.py +3 -0
  1297. ragbench/vectorstore/methods/singlestore/metadata.json +53 -0
  1298. ragbench/vectorstore/methods/singlestore/requirements.txt +1 -0
  1299. ragbench/vectorstore/methods/singlestore/store.py +39 -0
  1300. ragbench/vectorstore/methods/sqlite/__init__.py +3 -0
  1301. ragbench/vectorstore/methods/sqlite/metadata.json +39 -0
  1302. ragbench/vectorstore/methods/sqlite/requirements.txt +0 -0
  1303. ragbench/vectorstore/methods/sqlite/store.py +150 -0
  1304. ragbench/vectorstore/methods/sqlite_vec/__init__.py +3 -0
  1305. ragbench/vectorstore/methods/sqlite_vec/metadata.json +40 -0
  1306. ragbench/vectorstore/methods/sqlite_vec/requirements.txt +1 -0
  1307. ragbench/vectorstore/methods/sqlite_vec/store.py +58 -0
  1308. ragbench/vectorstore/methods/turbopuffer/__init__.py +3 -0
  1309. ragbench/vectorstore/methods/turbopuffer/metadata.json +56 -0
  1310. ragbench/vectorstore/methods/turbopuffer/requirements.txt +0 -0
  1311. ragbench/vectorstore/methods/turbopuffer/store.py +65 -0
  1312. ragbench/vectorstore/methods/typesense/__init__.py +3 -0
  1313. ragbench/vectorstore/methods/typesense/metadata.json +55 -0
  1314. ragbench/vectorstore/methods/typesense/requirements.txt +0 -0
  1315. ragbench/vectorstore/methods/typesense/store.py +82 -0
  1316. ragbench/vectorstore/methods/upstash/__init__.py +3 -0
  1317. ragbench/vectorstore/methods/upstash/metadata.json +57 -0
  1318. ragbench/vectorstore/methods/upstash/requirements.txt +0 -0
  1319. ragbench/vectorstore/methods/upstash/store.py +52 -0
  1320. ragbench/vectorstore/methods/usearch/__init__.py +3 -0
  1321. ragbench/vectorstore/methods/usearch/metadata.json +39 -0
  1322. ragbench/vectorstore/methods/usearch/requirements.txt +2 -0
  1323. ragbench/vectorstore/methods/usearch/store.py +29 -0
  1324. ragbench/vectorstore/methods/weaviate/__init__.py +3 -0
  1325. ragbench/vectorstore/methods/weaviate/metadata.json +58 -0
  1326. ragbench/vectorstore/methods/weaviate/requirements.txt +0 -0
  1327. ragbench/vectorstore/methods/weaviate/store.py +99 -0
  1328. ragbench/vectorstore/models.py +135 -0
  1329. ragbench/vectorstore/registry.py +115 -0
  1330. ragbench/vectorstore/store.py +97 -0
  1331. ragbench_ai-0.4.0.dist-info/METADATA +546 -0
  1332. ragbench_ai-0.4.0.dist-info/RECORD +1336 -0
  1333. ragbench_ai-0.4.0.dist-info/WHEEL +5 -0
  1334. ragbench_ai-0.4.0.dist-info/entry_points.txt +2 -0
  1335. ragbench_ai-0.4.0.dist-info/licenses/LICENSE +201 -0
  1336. ragbench_ai-0.4.0.dist-info/top_level.txt +1 -0
ragbench/__init__.py ADDED
@@ -0,0 +1,26 @@
1
+ """RAGBench: Layered RAG framework with independent, auto-chainable components.
2
+
3
+ Usage:
4
+ from ragbench import OmniRAG, ask
5
+ from ragbench.pipeline import auto
6
+ from ragbench.embedding import embed
7
+ from ragbench.retrieval import retrieve
8
+ from ragbench.generation import Generator
9
+ from ragbench.evaluation import Evaluator
10
+ """
11
+
12
+ __version__ = "0.4.0"
13
+
14
+ # Export main pipeline functions
15
+ from ragbench.pipeline import ask, auto, OmniPipeline
16
+
17
+ # Alias OmniPipeline as OmniRAG for convenience
18
+ OmniRAG = OmniPipeline
19
+
20
+ __all__ = [
21
+ "__version__",
22
+ "ask",
23
+ "auto",
24
+ "OmniPipeline",
25
+ "OmniRAG",
26
+ ]
@@ -0,0 +1,50 @@
1
+ # Chunking
2
+
3
+ The chunker consumes OmniLoad's canonical `Document` output. It does not replace or change the ingestion connectors.
4
+
5
+ ## Automatic mode
6
+
7
+ ```python
8
+ from core.loader import Loader
9
+
10
+ loader = Loader(connectors_dir="connectors")
11
+ chunks = loader.process_files(["report.pdf"], auto=True)
12
+
13
+ result = loader.process_files(
14
+ ["report.pdf"],
15
+ auto=True,
16
+ chunking_method="Recursive Character Chunking",
17
+ chunking_options={"chunk_size": 1200, "overlap": 120},
18
+ include_documents=True,
19
+ )
20
+ documents = result["documents"]
21
+ chunks = result["chunks"]
22
+ ```
23
+
24
+ `loader.load(path, auto=True)` is the single-file equivalent. For registered sources, `loader.process_connector(name, config, auto=True)` ingests and chunks the connector's output. SQLite accepts `{"file_path": "data.sqlite"}`; legacy database connectors accept connection settings such as `{"host": ..., "database": ..., "user": ..., "password": ...}`. Connector-specific drivers must be installed in the active Python environment for this direct API.
25
+
26
+ Without `auto=True`, `load()` and `process_files()` retain their document-returning behavior, and `process_connector()` returns normalized documents.
27
+
28
+ ## Manual mode
29
+
30
+ ```python
31
+ from core.loader import Loader
32
+ from chunking import Chunker
33
+
34
+ loader = Loader(connectors_dir="connectors")
35
+ documents = loader.process_files(["report.pdf"])
36
+ chunker = Chunker("Sentence Chunking", chunk_size=800)
37
+ chunks = chunker.chunk_documents(documents)
38
+ ```
39
+
40
+ `Chunker.available_methods()` lists all registered strategy names. Each result is a `Chunk` with `id`, `document_id`, `content`, and provenance metadata, including the source document metadata and selected strategy.
41
+
42
+ ## Strategies and capabilities
43
+
44
+ Every named strategy has its own module under `chunking/methods/`, for example `fixed_character_chunking.py` and `json_object_chunking.py`. Each module exports a `Strategy` class. `chunking.methods` discovers these modules and registers their strategy names; `Chunker` dispatches each registered strategy to the shared parser and boundary engine. This keeps methods separately addressable without duplicating common parsing code.
45
+
46
+ The registry includes the requested general, semantic, hierarchical, structure-aware, Markdown, HTML/XML, JSON/YAML, document, table/database, code, conversation, audio/video/image, graph, and adaptive strategy names. Format-aware methods use structure available in document text and metadata. They cannot recover layout or binary-media details that an ingestion connector did not retain.
47
+
48
+ To add a strategy, create `chunking/methods/<method_name>.py`, subclass `ChunkingStrategy` as `Strategy`, and set its `name` and a supported `group`. The package discovers the file automatically.
49
+
50
+ Embedding-based boundaries accept `embedder(sentences)`, which must return one numeric vector per sentence. Similarity methods accept `similarity_fn(left, right)`. LLM semantic chunking accepts `llm_segmenter(document_text)`, returning chunk strings. Without these callbacks, semantic methods use deterministic sentence/size boundaries and mark chunks with `semantic_fallback=True`; no model is loaded implicitly.
@@ -0,0 +1,17 @@
1
+ """Independent chunking framework (no dependency on the ingestion layer).
2
+
3
+ from ragbench.chunking import chunk
4
+ chunks = chunk("some text", method="auto")
5
+ """
6
+ from typing import Any, List
7
+
8
+ from ragbench.chunking.chunker import CHUNKING_METHODS, REGISTRY, Chunker
9
+ from ragbench.chunking.models import Chunk
10
+
11
+
12
+ def chunk(data: Any, method: str = "auto", **options: Any) -> List[Chunk]:
13
+ """Chunk a string, dict, Document-like object, or list of them."""
14
+ return Chunker(method, **options).chunk(data)
15
+
16
+
17
+ __all__ = ["CHUNKING_METHODS", "REGISTRY", "Chunk", "Chunker", "chunk"]
@@ -0,0 +1,507 @@
1
+ from __future__ import annotations
2
+
3
+ import ast
4
+ import csv
5
+ import hashlib
6
+ import html.parser
7
+ import importlib
8
+ import io
9
+ import json
10
+ import re
11
+ import uuid
12
+ import xml.etree.ElementTree as ET
13
+ from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, cast
14
+
15
+ from ragbench.chunking.models import Chunk
16
+ from ragbench.chunking.registry import ChunkingRegistry
17
+
18
+
19
+ REGISTRY = ChunkingRegistry()
20
+ METHOD_REGISTRY = {name: REGISTRY.get(name) for name in REGISTRY.list_names()}
21
+ CHUNKING_METHODS: Dict[str, str] = REGISTRY.groups()
22
+
23
+
24
+ def _get(document: Any, key: str, default: Any = None) -> Any:
25
+ if isinstance(document, Mapping):
26
+ return cast(Mapping[str, Any], document).get(key, default)
27
+ return getattr(document, key, default)
28
+
29
+
30
+ def _plain_text(value: Any) -> str:
31
+ if value is None:
32
+ return ""
33
+ if isinstance(value, str):
34
+ return value
35
+ if isinstance(value, (dict, list, tuple)):
36
+ return json.dumps(value, ensure_ascii=False, default=str, indent=2)
37
+ return str(value)
38
+
39
+
40
+ def _sentences(text: str) -> List[str]:
41
+ return [part.strip() for part in re.split(r"(?<=[.!?])\s+|\n+", text) if part.strip()]
42
+
43
+
44
+ def _paragraphs(text: str) -> List[str]:
45
+ return [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
46
+
47
+
48
+ def _markdown_sections(text: str) -> List[Tuple[str, str]]:
49
+ sections: List[Tuple[str, str]] = []
50
+ title = "Document"
51
+ body: List[str] = []
52
+ for line in text.splitlines():
53
+ if re.match(r"^\s{0,3}#{1,6}\s+", line):
54
+ if body:
55
+ sections.append((title, "\n".join(body).strip()))
56
+ title = re.sub(r"^\s{0,3}#{1,6}\s+", "", line).strip()
57
+ body = [line]
58
+ else:
59
+ body.append(line)
60
+ if body:
61
+ sections.append((title, "\n".join(body).strip()))
62
+ return [(title, body) for title, body in sections if body]
63
+
64
+
65
+ class _HTMLTextExtractor(html.parser.HTMLParser):
66
+ def __init__(self) -> None:
67
+ super().__init__(convert_charrefs=True)
68
+ self.parts: List[str] = []
69
+ self.stack: List[str] = []
70
+
71
+ def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
72
+ _ = attrs
73
+ if tag in {"p", "div", "section", "article", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "br"}:
74
+ self.parts.append("\n")
75
+ self.stack.append(tag)
76
+
77
+ def handle_endtag(self, tag: str) -> None:
78
+ if tag in {"p", "div", "section", "article", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6"}:
79
+ self.parts.append("\n")
80
+ if tag in self.stack:
81
+ self.stack.reverse()
82
+ self.stack.remove(tag)
83
+ self.stack.reverse()
84
+
85
+ def handle_data(self, data: str) -> None:
86
+ if data.strip():
87
+ self.parts.append(data.strip())
88
+
89
+
90
+ class Chunker:
91
+ """Turn OmniLoad Documents into chunks while retaining source provenance.
92
+
93
+ Semantic/model-dependent methods accept optional ``similarity_fn``,
94
+ ``embedder``, or ``llm_segmenter`` callbacks. Without them, these methods
95
+ use deterministic text boundaries and mark the fallback in chunk metadata.
96
+ """
97
+
98
+ def __init__(
99
+ self,
100
+ method: str = "Automatic Chunker Selection",
101
+ chunk_size: int = 1000,
102
+ overlap: int = 100,
103
+ token_counter: Optional[Callable[[str], int]] = None,
104
+ similarity_fn: Optional[Callable[[str, str], float]] = None,
105
+ embedder: Optional[Callable[[Sequence[str]], Sequence[Any]]] = None,
106
+ llm_segmenter: Optional[Callable[[str], Sequence[str]]] = None,
107
+ query: Optional[str] = None,
108
+ **options: Any,
109
+ ) -> None:
110
+ if method.strip().lower() == "auto":
111
+ method = "Automatic Chunker Selection"
112
+ canonical = REGISTRY.resolve(method)
113
+ if chunk_size < 1:
114
+ raise ValueError("chunk_size must be greater than zero")
115
+ if overlap < 0:
116
+ raise ValueError("overlap cannot be negative")
117
+ self.method = canonical
118
+ self.strategy = METHOD_REGISTRY[canonical]
119
+ self.group = CHUNKING_METHODS[canonical]
120
+ self.chunk_size = chunk_size
121
+ self.overlap = min(overlap, chunk_size - 1)
122
+ self.token_counter: Callable[[str], int] = token_counter or _count_tokens
123
+ self.similarity_fn = similarity_fn
124
+ self.embedder = embedder
125
+ self.llm_segmenter = llm_segmenter
126
+ self.query = query
127
+ self.options: Dict[str, Any] = options
128
+
129
+ @classmethod
130
+ def available_methods(cls) -> List[str]:
131
+ return list(CHUNKING_METHODS)
132
+
133
+ def chunk_documents(self, documents: Iterable[Any]) -> List[Chunk]:
134
+ chunks: List[Chunk] = []
135
+ for document in documents:
136
+ chunks.extend(self.chunk_document(document))
137
+ return chunks
138
+
139
+ def chunk(self, data: Any, metadata: Optional[Dict[str, Any]] = None) -> List[Chunk]:
140
+ """Standalone entry point: accepts a str, dict, Document-like object, or a list of them."""
141
+ if isinstance(data, (list, tuple)):
142
+ return self.chunk_documents(cast(Sequence[Any], data))
143
+ if isinstance(data, str) and metadata:
144
+ data = {"content": data, "metadata": metadata}
145
+ return self.chunk_document(data)
146
+
147
+ def chunk_document(self, document: Any) -> List[Chunk]:
148
+ content = _plain_text(_get(document, "content", document))
149
+ document_id = str(_get(document, "id", "")) or str(uuid.uuid4())
150
+ metadata_value = _get(document, "metadata", {})
151
+ document_metadata = cast(Dict[str, Any], metadata_value) if isinstance(metadata_value, dict) else {}
152
+ source = str(_get(document, "source", "") or "")
153
+ method = self.method
154
+ group = self.group
155
+
156
+ if method == "Automatic Chunker Selection":
157
+ method, group = self._select_method(content, document_metadata)
158
+ if method == "Content-Type-Aware Chunking":
159
+ method, group = self._select_method(content, document_metadata)
160
+ if method == "Query-Aware Chunking" and self.query:
161
+ group = "semantic"
162
+ if method in {"Embedding-Based Semantic Chunking", "LLM-Based Semantic Chunking", "Agentic Chunking"}:
163
+ if method == "LLM-Based Semantic Chunking" and self.llm_segmenter:
164
+ proposed = self.llm_segmenter(content)
165
+ if proposed:
166
+ return self._make_chunks(proposed, document_id, document_metadata, source, method)
167
+
168
+ strategy = METHOD_REGISTRY[method]
169
+ segments, extra_metadata = strategy.split(self, content, document_metadata)
170
+ return self._make_chunks(segments, document_id, document_metadata, source, method, extra_metadata)
171
+
172
+ def _select_method(self, content: str, metadata: Dict[str, Any]) -> Tuple[str, str]:
173
+ content_type = str(metadata.get("content_type", metadata.get("format", ""))).lower()
174
+ if content_type in {"markdown", "md"} or re.search(r"^#{1,6}\s", content, re.M):
175
+ return "Markdown Heading Chunking", "markdown_heading"
176
+ if content_type in {"json", "application/json"} or content.lstrip().startswith(("{", "[")):
177
+ return "JSON Object Chunking", "json_object"
178
+ if content_type in {"html", "xml"} or re.search(r"<(html|body|article|section|root)\b", content, re.I):
179
+ return "DOM-Based Chunking", "html_dom"
180
+ if "table" in content_type or metadata.get("table"):
181
+ return "Record-Based Chunking", "row"
182
+ return "Recursive Character Chunking", "recursive"
183
+
184
+ def _split(self, text: str, group: str, metadata: Dict[str, Any]) -> Tuple[List[str], Dict[str, Any]]:
185
+ if not text.strip():
186
+ return [], {}
187
+ if group == "fixed_character":
188
+ return self._fixed(text, self.chunk_size, self.overlap), {}
189
+ if group == "fixed_token":
190
+ return self._token_chunks(text), {}
191
+ if group == "sentence":
192
+ return self._group_units(_sentences(text), lambda item: len(item), self.chunk_size), {}
193
+ if group == "paragraph":
194
+ return _paragraphs(text), {}
195
+ if group == "recursive":
196
+ return self._recursive(text), {}
197
+ if group in {"semantic", "audio_semantic"}:
198
+ segments, fallback = self._semantic(text)
199
+ return segments, {"semantic_fallback": fallback} if fallback else {}
200
+ if group == "hierarchical":
201
+ return self._hierarchical(text), {"hierarchy": ["document", "section", "paragraph"]}
202
+ if group in {"heading", "markdown_heading", "markdown_code", "markdown_table", "pdf_section", "docx_heading"}:
203
+ if group == "markdown_code":
204
+ return self._markdown_blocks(text, r"```[\s\S]*?```"), {}
205
+ if group == "markdown_table":
206
+ return self._markdown_blocks(text, r"(?:^\|.*\|\s*\n){2,}"), {}
207
+ return [body for _, body in _markdown_sections(text)], {}
208
+ if group == "list":
209
+ return self._list_blocks(text), {}
210
+ if group in {"block", "docx_paragraph", "page", "pdf_layout", "pptx_slide", "video_scene", "repository"}:
211
+ delimiter = "\f" if group == "page" else r"\n(?=\s*(?:```|#{1,6}\s|(?:def|class)\s))"
212
+ if group == "pptx_slide":
213
+ delimiter = r"\n(?=(?:Slide\s+\d+|={3,}\s*Slide))"
214
+ if group == "video_scene":
215
+ delimiter = r"\n(?=(?:Scene|Shot)\s+\d+)"
216
+ parts = re.split(delimiter, text) if group == "page" else re.split(delimiter, text)
217
+ return [part.strip() for part in parts if part.strip()], {}
218
+ if group in {"html_dom", "html_section"}:
219
+ parser = _HTMLTextExtractor()
220
+ parser.feed(text)
221
+ plain = " ".join(" ".join(parser.parts).split())
222
+ segments = _paragraphs(plain) or [plain]
223
+ return segments, {"structure": group}
224
+ if group == "xml_tree":
225
+ try:
226
+ root = ET.fromstring(text)
227
+ elements = [" ".join(part.strip() for part in element.itertext() if part.strip()) for element in root.iter()]
228
+ return [item for item in elements if item], {"structure": "xml"}
229
+ except ET.ParseError:
230
+ return self._recursive(text), {"parse_fallback": True}
231
+ if group in {"json_object", "json_array", "nested", "yaml", "schema"}:
232
+ return self._structured(text, group), {"structure": group}
233
+ if group in {"row", "column", "table", "pdf_table", "docx_table"}:
234
+ return self._table_chunks(text, group), {"structure": group}
235
+ if group == "sql":
236
+ return [part.strip() for part in re.split(r";\s*(?:\n|$)", text) if part.strip()], {}
237
+ if group in {"ast", "dependency"}:
238
+ return self._code_chunks(text, group), {"structure": "python_ast"}
239
+ if group in {"message", "thread", "speaker", "time", "transcript"}:
240
+ return self._communication_chunks(text, group), {"structure": group}
241
+ if group in {"image_ocr", "image_region"}:
242
+ regions: Any = metadata.get("regions") or metadata.get("ocr")
243
+ if isinstance(regions, list) and regions:
244
+ region_items = cast(List[Any], regions)
245
+ return [_plain_text(region) for region in region_items], {"structure": "image_regions"}
246
+ return self._recursive(text), {"structure_fallback": "text"}
247
+ if group.startswith("graph_"):
248
+ return self._graph_chunks(text, group), {"structure": group}
249
+ return self._recursive(text), {}
250
+
251
+ def _fixed(self, text: str, size: int, overlap: int) -> List[str]:
252
+ output: List[str] = []
253
+ start = 0
254
+ while start < len(text):
255
+ end = min(start + size, len(text))
256
+ if end < len(text):
257
+ boundary = text.rfind(" ", start, end)
258
+ if boundary > start:
259
+ end = boundary
260
+ output.append(text[start:end].strip())
261
+ if end >= len(text):
262
+ break
263
+ start = max(start + 1, end - overlap)
264
+ return [part for part in output if part]
265
+
266
+ def _token_chunks(self, text: str) -> List[str]:
267
+ words = text.split()
268
+ output: List[str] = []
269
+ start = 0
270
+ while start < len(words):
271
+ end = start
272
+ while end < len(words) and self.token_counter(" ".join(words[start:end + 1])) <= self.chunk_size:
273
+ end += 1
274
+ if end == start:
275
+ end += 1
276
+ output.append(" ".join(words[start:end]))
277
+ if end >= len(words):
278
+ break
279
+ start = max(start + 1, end - self.overlap)
280
+ return output
281
+
282
+ def _recursive(self, text: str) -> List[str]:
283
+ output: List[str] = []
284
+ for paragraph in _paragraphs(text):
285
+ if len(paragraph) <= self.chunk_size:
286
+ output.append(paragraph)
287
+ else:
288
+ units = _sentences(paragraph)
289
+ output.extend(self._group_units(units, len, self.chunk_size) if units else self._fixed(paragraph, self.chunk_size, self.overlap))
290
+ return output
291
+
292
+ @staticmethod
293
+ def _group_units(units: Sequence[str], measure: Callable[[str], int], size: int) -> List[str]:
294
+ output: List[str] = []
295
+ current: List[str] = []
296
+ length = 0
297
+ for unit in units:
298
+ unit_length = measure(unit)
299
+ if current and length + unit_length + 1 > size:
300
+ output.append(" ".join(current))
301
+ current = []
302
+ length = 0
303
+ current.append(unit)
304
+ length += unit_length + 1
305
+ if current:
306
+ output.append(" ".join(current))
307
+ return output
308
+
309
+ def _semantic(self, text: str) -> Tuple[List[str], bool]:
310
+ units = _sentences(text)
311
+ if not units:
312
+ return [], False
313
+ embeddings = list(self.embedder(units)) if self.embedder else []
314
+ output: List[str] = []
315
+ current = [units[0]]
316
+ for index, (previous, unit) in enumerate(zip(units, units[1:])):
317
+ if self.similarity_fn:
318
+ similarity = self.similarity_fn(previous, unit)
319
+ elif len(embeddings) == len(units):
320
+ left, right = embeddings[index], embeddings[index + 1]
321
+ dot = sum(float(a) * float(b) for a, b in zip(left, right))
322
+ left_norm = sum(float(value) ** 2 for value in left) ** 0.5
323
+ right_norm = sum(float(value) ** 2 for value in right) ** 0.5
324
+ similarity = dot / (left_norm * right_norm) if left_norm and right_norm else 0.0
325
+ else:
326
+ similarity = 1.0
327
+ similar = similarity >= self.options.get("similarity_threshold", 0.5)
328
+ if not similar or len(" ".join(current)) + len(unit) > self.chunk_size:
329
+ output.append(" ".join(current))
330
+ current = []
331
+ current.append(unit)
332
+ if current:
333
+ output.append(" ".join(current))
334
+ return output, self.similarity_fn is None and not embeddings and self.llm_segmenter is None
335
+
336
+ def _hierarchical(self, text: str) -> List[str]:
337
+ output: List[str] = []
338
+ for _, section in _markdown_sections(text):
339
+ output.extend(_paragraphs(section) or [section])
340
+ return output
341
+
342
+ def _markdown_blocks(self, text: str, pattern: str) -> List[str]:
343
+ matches = list(re.finditer(pattern, text, re.M))
344
+ if not matches:
345
+ return self._recursive(text)
346
+ parts: List[str] = []
347
+ cursor = 0
348
+ for match in matches:
349
+ if match.start() > cursor:
350
+ parts.extend(_paragraphs(text[cursor:match.start()]))
351
+ parts.append(match.group(0).strip())
352
+ cursor = match.end()
353
+ if cursor < len(text):
354
+ parts.extend(_paragraphs(text[cursor:]))
355
+ return [part for part in parts if part]
356
+
357
+ @staticmethod
358
+ def _list_blocks(text: str) -> List[str]:
359
+ blocks: List[str] = []
360
+ for line in text.splitlines():
361
+ if re.match(r"^\s*(?:[-*+] |\d+[.)] )", line) and blocks:
362
+ blocks[-1] += "\n" + line
363
+ elif re.match(r"^\s*(?:[-*+] |\d+[.)] )", line):
364
+ blocks.append(line)
365
+ elif line.strip():
366
+ blocks.append(line.strip())
367
+ return blocks
368
+
369
+ def _structured(self, text: str, group: str) -> List[str]:
370
+ try:
371
+ if group == "yaml":
372
+ yaml_module: Any = importlib.import_module("yaml")
373
+ value: Any = yaml_module.safe_load(text)
374
+ else:
375
+ value = json.loads(text)
376
+ except Exception: # yaml.YAMLError is not a ValueError; any parse failure -> text fallback
377
+ return self._recursive(text)
378
+ if group == "yaml" and not isinstance(value, (dict, list)):
379
+ return self._recursive(text) # plain prose that merely parses as a YAML scalar
380
+ if isinstance(value, list):
381
+ items = cast(List[Any], value)
382
+ return [json.dumps(item, ensure_ascii=False, default=str) for item in items] or ["[]"]
383
+ if isinstance(value, dict):
384
+ mapping = cast(Dict[str, Any], value)
385
+ if group in {"json_object", "nested", "schema"}:
386
+ return [json.dumps({key: item}, ensure_ascii=False, default=str) for key, item in mapping.items()] or ["{}"]
387
+ return [json.dumps(value, ensure_ascii=False, default=str)]
388
+ return [json.dumps(value, ensure_ascii=False, default=str)]
389
+
390
+ def _table_chunks(self, text: str, group: str) -> List[str]:
391
+ lines = [line for line in text.splitlines() if line.strip()]
392
+ if not lines:
393
+ return []
394
+ try:
395
+ rows: List[List[str]] = list(csv.reader(io.StringIO("\n".join(lines))))
396
+ except csv.Error:
397
+ rows = [line.split("\t") for line in lines]
398
+ if group == "column":
399
+ return ["\n".join(row[index] for row in rows if index < len(row)) for index in range(max(map(len, rows)))]
400
+ if group in {"table", "schema", "pdf_table", "docx_table"}:
401
+ return ["\n".join(lines)]
402
+ width = max(1, int(self.options.get("rows_per_chunk", 10)))
403
+ header = rows[0] if group == "table" else []
404
+ data_rows = rows[1:] if header else rows
405
+ output: List[str] = []
406
+ for index in range(0, len(data_rows), width):
407
+ group_rows = ([header] if header else []) + data_rows[index:index + width]
408
+ output.append("\n".join(",".join(row) for row in group_rows))
409
+ return output
410
+
411
+ def _code_chunks(self, text: str, group: str) -> List[str]:
412
+ try:
413
+ tree = ast.parse(text)
414
+ except SyntaxError:
415
+ return self._recursive(text)
416
+ nodes = []
417
+ if group == "dependency":
418
+ nodes = [node for node in ast.walk(tree) if isinstance(node, (ast.Import, ast.ImportFrom))]
419
+ if not nodes:
420
+ return [text]
421
+ else:
422
+ nodes = [node for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))]
423
+ if not nodes:
424
+ return [text]
425
+ return [ast.get_source_segment(text, node) or "" for node in nodes if ast.get_source_segment(text, node)]
426
+
427
+ def _communication_chunks(self, text: str, group: str) -> List[str]:
428
+ messages = [line.strip() for line in text.splitlines() if line.strip()]
429
+ if group == "speaker":
430
+ speakers: Dict[str, List[str]] = {}
431
+ for message in messages:
432
+ match = re.match(r"\[?([^:\]]+)\]?:\s*(.*)", message)
433
+ key, body = (match.group(1), match.group(2)) if match else ("unknown", message)
434
+ speakers.setdefault(key, []).append(body)
435
+ return [f"{speaker}: " + "\n".join(items) for speaker, items in speakers.items()]
436
+ if group == "time":
437
+ window = int(self.options.get("window_seconds", 60))
438
+ chunks: Dict[int, List[str]] = {}
439
+ for message in messages:
440
+ match = re.search(r"(?:\[)?(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\])?", message)
441
+ seconds = int(match.group(1)) * 60 + int(match.group(2)) if match else 0
442
+ chunks.setdefault(seconds // max(1, window), []).append(message)
443
+ return ["\n".join(items) for items in chunks.values()]
444
+ if group == "thread":
445
+ return [part.strip() for part in re.split(r"\n(?=Thread\s|Subject:)", text) if part.strip()]
446
+ return ["\n".join(messages[index:index + 10]) for index in range(0, len(messages), 10)]
447
+
448
+ def _graph_chunks(self, text: str, group: str) -> List[str]:
449
+ try:
450
+ parsed: Any = json.loads(text)
451
+ except ValueError:
452
+ return self._recursive(text)
453
+ if not isinstance(parsed, dict):
454
+ return [text]
455
+ graph = cast(Dict[str, Any], parsed)
456
+ nodes_value: Any = graph.get("nodes", [])
457
+ edges_value: Any = graph.get("edges", [])
458
+ nodes = cast(List[Any], nodes_value) if isinstance(nodes_value, list) else []
459
+ edges = cast(List[Any], edges_value) if isinstance(edges_value, list) else []
460
+ values: List[Any]
461
+ if group == "graph_node":
462
+ values = nodes
463
+ elif group == "graph_edge":
464
+ values = edges
465
+ else:
466
+ values = nodes or edges
467
+ return [json.dumps(value, ensure_ascii=False, default=str) for value in values] or [text]
468
+
469
+ def _make_chunks(
470
+ self,
471
+ segments: Iterable[Any],
472
+ document_id: str,
473
+ document_metadata: Dict[str, Any],
474
+ source: str,
475
+ method: str,
476
+ extra_metadata: Optional[Dict[str, Any]] = None,
477
+ ) -> List[Chunk]:
478
+ chunks: List[Chunk] = []
479
+ parent_id = document_metadata.get("parent_id", document_id)
480
+ for index, segment in enumerate(segments):
481
+ content = _plain_text(segment).strip()
482
+ if not content:
483
+ continue
484
+ metadata: Dict[str, Any] = {
485
+ "chunk_index": len(chunks),
486
+ "source": source,
487
+ "strategy": method,
488
+ "document_metadata": document_metadata,
489
+ "parent_id": parent_id,
490
+ }
491
+ if extra_metadata:
492
+ metadata.update(extra_metadata)
493
+ metadata["chunk_index"] = len(chunks)
494
+ chunks.append(Chunk(
495
+ id=hashlib.sha256(f"{document_id}:{method}:{index}:{content}".encode("utf-8")).hexdigest(),
496
+ document_id=document_id,
497
+ content=content,
498
+ metadata=metadata,
499
+ ))
500
+ count = len(chunks)
501
+ for chunk in chunks:
502
+ chunk.metadata["chunk_count"] = count
503
+ return chunks
504
+
505
+
506
+ def _count_tokens(text: str) -> int:
507
+ return len(text.split())
@@ -0,0 +1,16 @@
1
+ """Chunking methods: one folder per method (see chunking/registry.py)."""
2
+ from typing import Dict
3
+
4
+ from ragbench.chunking.methods.base import ChunkingStrategy
5
+
6
+ __all__ = ["ChunkingStrategy", "METHOD_REGISTRY"]
7
+
8
+
9
+ def __getattr__(attr: str):
10
+ # Lazy so that importing a single method folder never triggers a full scan.
11
+ if attr == "METHOD_REGISTRY":
12
+ from ragbench.chunking.registry import ChunkingRegistry
13
+
14
+ registry = ChunkingRegistry()
15
+ return {n: registry.get(n) for n in registry.list_names()}
16
+ raise AttributeError(attr)
@@ -0,0 +1,3 @@
1
+ from .strategy import Strategy
2
+
3
+ __all__ = ["Strategy"]
@@ -0,0 +1,10 @@
1
+ {
2
+ "name": "Adaptive Chunking",
3
+ "slug": "adaptive_chunking",
4
+ "group": "semantic",
5
+ "category": "semantic",
6
+ "extensions": [],
7
+ "description": "Adaptive Chunking (semantic / semantic implementation)",
8
+ "entry_point": "strategy.Strategy",
9
+ "requires_callbacks": []
10
+ }
@@ -0,0 +1,6 @@
1
+ from ragbench.chunking.methods.base import ChunkingStrategy
2
+
3
+
4
+ class Strategy(ChunkingStrategy):
5
+ name = "Adaptive Chunking"
6
+ group = "semantic"
@@ -0,0 +1,3 @@
1
+ from .strategy import Strategy
2
+
3
+ __all__ = ["Strategy"]
@@ -0,0 +1,10 @@
1
+ {
2
+ "name": "Agentic Chunking",
3
+ "slug": "agentic_chunking",
4
+ "group": "semantic",
5
+ "category": "semantic",
6
+ "extensions": [],
7
+ "description": "Agentic Chunking (semantic / semantic implementation)",
8
+ "entry_point": "strategy.Strategy",
9
+ "requires_callbacks": []
10
+ }
@@ -0,0 +1,6 @@
1
+ from ragbench.chunking.methods.base import ChunkingStrategy
2
+
3
+
4
+ class Strategy(ChunkingStrategy):
5
+ name = "Agentic Chunking"
6
+ group = "semantic"
@@ -0,0 +1,3 @@
1
+ from .strategy import Strategy
2
+
3
+ __all__ = ["Strategy"]