ragbench-ai 0.4.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- ragbench/__init__.py +26 -0
- ragbench/chunking/README.md +50 -0
- ragbench/chunking/__init__.py +17 -0
- ragbench/chunking/chunker.py +507 -0
- ragbench/chunking/methods/__init__.py +16 -0
- ragbench/chunking/methods/adaptive_chunking/__init__.py +3 -0
- ragbench/chunking/methods/adaptive_chunking/metadata.json +10 -0
- ragbench/chunking/methods/adaptive_chunking/strategy.py +6 -0
- ragbench/chunking/methods/agentic_chunking/__init__.py +3 -0
- ragbench/chunking/methods/agentic_chunking/metadata.json +10 -0
- ragbench/chunking/methods/agentic_chunking/strategy.py +6 -0
- ragbench/chunking/methods/ast_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/ast_based_chunking/metadata.json +12 -0
- ragbench/chunking/methods/ast_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/automatic_chunker_selection/__init__.py +3 -0
- ragbench/chunking/methods/automatic_chunker_selection/metadata.json +10 -0
- ragbench/chunking/methods/automatic_chunker_selection/strategy.py +6 -0
- ragbench/chunking/methods/base.py +19 -0
- ragbench/chunking/methods/block_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/block_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/block_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/class_level_chunking/__init__.py +3 -0
- ragbench/chunking/methods/class_level_chunking/metadata.json +12 -0
- ragbench/chunking/methods/class_level_chunking/strategy.py +6 -0
- ragbench/chunking/methods/code_block_chunking/__init__.py +3 -0
- ragbench/chunking/methods/code_block_chunking/metadata.json +10 -0
- ragbench/chunking/methods/code_block_chunking/strategy.py +6 -0
- ragbench/chunking/methods/column_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/column_based_chunking/metadata.json +13 -0
- ragbench/chunking/methods/column_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/community_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/community_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/community_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/content_type_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/content_type_aware_chunking/metadata.json +10 -0
- ragbench/chunking/methods/content_type_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/contextual_chunking/__init__.py +3 -0
- ragbench/chunking/methods/contextual_chunking/metadata.json +10 -0
- ragbench/chunking/methods/contextual_chunking/strategy.py +6 -0
- ragbench/chunking/methods/conversation_turn_chunking/__init__.py +3 -0
- ragbench/chunking/methods/conversation_turn_chunking/metadata.json +10 -0
- ragbench/chunking/methods/conversation_turn_chunking/strategy.py +6 -0
- ragbench/chunking/methods/document_section_paragraph_chunking/__init__.py +3 -0
- ragbench/chunking/methods/document_section_paragraph_chunking/metadata.json +10 -0
- ragbench/chunking/methods/document_section_paragraph_chunking/strategy.py +6 -0
- ragbench/chunking/methods/docx_heading_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/docx_heading_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/docx_heading_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/docx_paragraph_chunking/__init__.py +3 -0
- ragbench/chunking/methods/docx_paragraph_chunking/metadata.json +14 -0
- ragbench/chunking/methods/docx_paragraph_chunking/strategy.py +6 -0
- ragbench/chunking/methods/docx_table_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/docx_table_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/docx_table_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/dom_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/dom_based_chunking/metadata.json +13 -0
- ragbench/chunking/methods/dom_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/dynamic_chunk_size_chunking/__init__.py +3 -0
- ragbench/chunking/methods/dynamic_chunk_size_chunking/metadata.json +10 -0
- ragbench/chunking/methods/dynamic_chunk_size_chunking/strategy.py +6 -0
- ragbench/chunking/methods/edge_relationship_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/edge_relationship_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/edge_relationship_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/embedding_based_semantic_chunking/__init__.py +3 -0
- ragbench/chunking/methods/embedding_based_semantic_chunking/metadata.json +12 -0
- ragbench/chunking/methods/embedding_based_semantic_chunking/strategy.py +6 -0
- ragbench/chunking/methods/fixed_character_chunking/__init__.py +3 -0
- ragbench/chunking/methods/fixed_character_chunking/metadata.json +10 -0
- ragbench/chunking/methods/fixed_character_chunking/strategy.py +6 -0
- ragbench/chunking/methods/fixed_token_chunking/__init__.py +3 -0
- ragbench/chunking/methods/fixed_token_chunking/metadata.json +10 -0
- ragbench/chunking/methods/fixed_token_chunking/strategy.py +6 -0
- ragbench/chunking/methods/function_level_chunking/__init__.py +3 -0
- ragbench/chunking/methods/function_level_chunking/metadata.json +12 -0
- ragbench/chunking/methods/function_level_chunking/strategy.py +6 -0
- ragbench/chunking/methods/header_row_chunking/__init__.py +3 -0
- ragbench/chunking/methods/header_row_chunking/metadata.json +14 -0
- ragbench/chunking/methods/header_row_chunking/strategy.py +6 -0
- ragbench/chunking/methods/heading_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/heading_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/heading_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/hierarchical_chunking/__init__.py +3 -0
- ragbench/chunking/methods/hierarchical_chunking/metadata.json +10 -0
- ragbench/chunking/methods/hierarchical_chunking/strategy.py +6 -0
- ragbench/chunking/methods/html_element_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/html_element_based_chunking/metadata.json +13 -0
- ragbench/chunking/methods/html_element_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/html_section_chunking/__init__.py +3 -0
- ragbench/chunking/methods/html_section_chunking/metadata.json +13 -0
- ragbench/chunking/methods/html_section_chunking/strategy.py +6 -0
- ragbench/chunking/methods/import_dependency_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/import_dependency_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/import_dependency_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/json_array_record_chunking/__init__.py +3 -0
- ragbench/chunking/methods/json_array_record_chunking/metadata.json +13 -0
- ragbench/chunking/methods/json_array_record_chunking/strategy.py +6 -0
- ragbench/chunking/methods/json_nested_object_chunking/__init__.py +3 -0
- ragbench/chunking/methods/json_nested_object_chunking/metadata.json +12 -0
- ragbench/chunking/methods/json_nested_object_chunking/strategy.py +6 -0
- ragbench/chunking/methods/json_object_chunking/__init__.py +3 -0
- ragbench/chunking/methods/json_object_chunking/metadata.json +12 -0
- ragbench/chunking/methods/json_object_chunking/strategy.py +6 -0
- ragbench/chunking/methods/jsonpath_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/jsonpath_based_chunking/metadata.json +12 -0
- ragbench/chunking/methods/jsonpath_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/key_value_chunking/__init__.py +3 -0
- ragbench/chunking/methods/key_value_chunking/metadata.json +12 -0
- ragbench/chunking/methods/key_value_chunking/strategy.py +6 -0
- ragbench/chunking/methods/layout_aware_image_chunking/__init__.py +3 -0
- ragbench/chunking/methods/layout_aware_image_chunking/metadata.json +16 -0
- ragbench/chunking/methods/layout_aware_image_chunking/strategy.py +6 -0
- ragbench/chunking/methods/list_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/list_aware_chunking/metadata.json +10 -0
- ragbench/chunking/methods/list_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/llm_based_semantic_chunking/__init__.py +3 -0
- ragbench/chunking/methods/llm_based_semantic_chunking/metadata.json +12 -0
- ragbench/chunking/methods/llm_based_semantic_chunking/strategy.py +6 -0
- ragbench/chunking/methods/markdown_code_block_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/markdown_code_block_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/markdown_code_block_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/markdown_heading_chunking/__init__.py +3 -0
- ragbench/chunking/methods/markdown_heading_chunking/metadata.json +13 -0
- ragbench/chunking/methods/markdown_heading_chunking/strategy.py +6 -0
- ragbench/chunking/methods/markdown_section_chunking/__init__.py +3 -0
- ragbench/chunking/methods/markdown_section_chunking/metadata.json +13 -0
- ragbench/chunking/methods/markdown_section_chunking/strategy.py +6 -0
- ragbench/chunking/methods/markdown_table_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/markdown_table_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/markdown_table_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/message_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/message_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/message_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/method_level_chunking/__init__.py +3 -0
- ragbench/chunking/methods/method_level_chunking/metadata.json +12 -0
- ragbench/chunking/methods/method_level_chunking/strategy.py +6 -0
- ragbench/chunking/methods/module_level_chunking/__init__.py +3 -0
- ragbench/chunking/methods/module_level_chunking/metadata.json +12 -0
- ragbench/chunking/methods/module_level_chunking/strategy.py +6 -0
- ragbench/chunking/methods/multi_resolution_chunking/__init__.py +3 -0
- ragbench/chunking/methods/multi_resolution_chunking/metadata.json +10 -0
- ragbench/chunking/methods/multi_resolution_chunking/strategy.py +6 -0
- ragbench/chunking/methods/multi_strategy_chunking/__init__.py +3 -0
- ragbench/chunking/methods/multi_strategy_chunking/metadata.json +10 -0
- ragbench/chunking/methods/multi_strategy_chunking/strategy.py +6 -0
- ragbench/chunking/methods/multimodal_video_chunking/__init__.py +3 -0
- ragbench/chunking/methods/multimodal_video_chunking/metadata.json +15 -0
- ragbench/chunking/methods/multimodal_video_chunking/strategy.py +6 -0
- ragbench/chunking/methods/neighborhood_chunking/__init__.py +3 -0
- ragbench/chunking/methods/neighborhood_chunking/metadata.json +10 -0
- ragbench/chunking/methods/neighborhood_chunking/strategy.py +6 -0
- ragbench/chunking/methods/nested_document_chunking/__init__.py +3 -0
- ragbench/chunking/methods/nested_document_chunking/metadata.json +12 -0
- ragbench/chunking/methods/nested_document_chunking/strategy.py +6 -0
- ragbench/chunking/methods/node_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/node_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/node_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/ocr_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/ocr_based_chunking/metadata.json +16 -0
- ragbench/chunking/methods/ocr_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/page_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/page_based_chunking/metadata.json +12 -0
- ragbench/chunking/methods/page_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/paragraph_chunking/__init__.py +3 -0
- ragbench/chunking/methods/paragraph_chunking/metadata.json +10 -0
- ragbench/chunking/methods/paragraph_chunking/strategy.py +6 -0
- ragbench/chunking/methods/parent_child_chunking/__init__.py +3 -0
- ragbench/chunking/methods/parent_child_chunking/metadata.json +10 -0
- ragbench/chunking/methods/parent_child_chunking/strategy.py +6 -0
- ragbench/chunking/methods/pdf_heading_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/pdf_heading_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/pdf_heading_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/pdf_layout_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/pdf_layout_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/pdf_layout_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/pdf_multi_column_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/pdf_multi_column_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/pdf_multi_column_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/pdf_section_chunking/__init__.py +3 -0
- ragbench/chunking/methods/pdf_section_chunking/metadata.json +12 -0
- ragbench/chunking/methods/pdf_section_chunking/strategy.py +6 -0
- ragbench/chunking/methods/pdf_table_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/pdf_table_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/pdf_table_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/pptx_element_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/pptx_element_aware_chunking/metadata.json +13 -0
- ragbench/chunking/methods/pptx_element_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/pptx_slide_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/pptx_slide_based_chunking/metadata.json +13 -0
- ragbench/chunking/methods/pptx_slide_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/proposition_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/proposition_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/proposition_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/query_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/query_aware_chunking/metadata.json +10 -0
- ragbench/chunking/methods/query_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/record_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/record_based_chunking/metadata.json +16 -0
- ragbench/chunking/methods/record_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/recursive_character_chunking/__init__.py +3 -0
- ragbench/chunking/methods/recursive_character_chunking/metadata.json +10 -0
- ragbench/chunking/methods/recursive_character_chunking/strategy.py +6 -0
- ragbench/chunking/methods/recursive_token_chunking/__init__.py +3 -0
- ragbench/chunking/methods/recursive_token_chunking/metadata.json +10 -0
- ragbench/chunking/methods/recursive_token_chunking/strategy.py +6 -0
- ragbench/chunking/methods/region_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/region_based_chunking/metadata.json +16 -0
- ragbench/chunking/methods/region_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/repository_hierarchy_chunking/__init__.py +3 -0
- ragbench/chunking/methods/repository_hierarchy_chunking/metadata.json +12 -0
- ragbench/chunking/methods/repository_hierarchy_chunking/strategy.py +6 -0
- ragbench/chunking/methods/row_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/row_based_chunking/metadata.json +16 -0
- ragbench/chunking/methods/row_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/row_group_chunking/__init__.py +3 -0
- ragbench/chunking/methods/row_group_chunking/metadata.json +16 -0
- ragbench/chunking/methods/row_group_chunking/strategy.py +6 -0
- ragbench/chunking/methods/scene_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/scene_based_chunking/metadata.json +15 -0
- ragbench/chunking/methods/scene_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/schema_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/schema_aware_chunking/metadata.json +10 -0
- ragbench/chunking/methods/schema_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/schema_aware_table_chunking/__init__.py +3 -0
- ragbench/chunking/methods/schema_aware_table_chunking/metadata.json +10 -0
- ragbench/chunking/methods/schema_aware_table_chunking/strategy.py +6 -0
- ragbench/chunking/methods/section_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/section_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/section_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/semantic_audio_chunking/__init__.py +3 -0
- ragbench/chunking/methods/semantic_audio_chunking/metadata.json +15 -0
- ragbench/chunking/methods/semantic_audio_chunking/strategy.py +6 -0
- ragbench/chunking/methods/semantic_chunking/__init__.py +3 -0
- ragbench/chunking/methods/semantic_chunking/metadata.json +10 -0
- ragbench/chunking/methods/semantic_chunking/strategy.py +6 -0
- ragbench/chunking/methods/sentence_based_audio_chunking/__init__.py +3 -0
- ragbench/chunking/methods/sentence_based_audio_chunking/metadata.json +10 -0
- ragbench/chunking/methods/sentence_based_audio_chunking/strategy.py +6 -0
- ragbench/chunking/methods/sentence_chunking/__init__.py +3 -0
- ragbench/chunking/methods/sentence_chunking/metadata.json +10 -0
- ragbench/chunking/methods/sentence_chunking/strategy.py +6 -0
- ragbench/chunking/methods/sentence_token_hybrid_chunking/__init__.py +3 -0
- ragbench/chunking/methods/sentence_token_hybrid_chunking/metadata.json +10 -0
- ragbench/chunking/methods/sentence_token_hybrid_chunking/strategy.py +6 -0
- ragbench/chunking/methods/sentence_window_chunking/__init__.py +3 -0
- ragbench/chunking/methods/sentence_window_chunking/metadata.json +10 -0
- ragbench/chunking/methods/sentence_window_chunking/strategy.py +6 -0
- ragbench/chunking/methods/shot_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/shot_based_chunking/metadata.json +15 -0
- ragbench/chunking/methods/shot_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/silence_voice_activity_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/silence_voice_activity_based_chunking/metadata.json +15 -0
- ragbench/chunking/methods/silence_voice_activity_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/similarity_boundary_chunking/__init__.py +3 -0
- ragbench/chunking/methods/similarity_boundary_chunking/metadata.json +10 -0
- ragbench/chunking/methods/similarity_boundary_chunking/strategy.py +6 -0
- ragbench/chunking/methods/sliding_window_chunking/__init__.py +3 -0
- ragbench/chunking/methods/sliding_window_chunking/metadata.json +10 -0
- ragbench/chunking/methods/sliding_window_chunking/strategy.py +6 -0
- ragbench/chunking/methods/speaker_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/speaker_aware_chunking/metadata.json +10 -0
- ragbench/chunking/methods/speaker_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/speaker_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/speaker_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/speaker_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/sql_ddl_aware_chunking/__init__.py +3 -0
- ragbench/chunking/methods/sql_ddl_aware_chunking/metadata.json +12 -0
- ragbench/chunking/methods/sql_ddl_aware_chunking/strategy.py +6 -0
- ragbench/chunking/methods/sql_query_result_chunking/__init__.py +3 -0
- ragbench/chunking/methods/sql_query_result_chunking/metadata.json +16 -0
- ragbench/chunking/methods/sql_query_result_chunking/strategy.py +6 -0
- ragbench/chunking/methods/sql_statement_chunking/__init__.py +3 -0
- ragbench/chunking/methods/sql_statement_chunking/metadata.json +12 -0
- ragbench/chunking/methods/sql_statement_chunking/strategy.py +6 -0
- ragbench/chunking/methods/structure_preserving_chunking/__init__.py +3 -0
- ragbench/chunking/methods/structure_preserving_chunking/metadata.json +10 -0
- ragbench/chunking/methods/structure_preserving_chunking/strategy.py +6 -0
- ragbench/chunking/methods/structure_semantic_hybrid_chunking/__init__.py +3 -0
- ragbench/chunking/methods/structure_semantic_hybrid_chunking/metadata.json +10 -0
- ragbench/chunking/methods/structure_semantic_hybrid_chunking/strategy.py +6 -0
- ragbench/chunking/methods/subgraph_chunking/__init__.py +3 -0
- ragbench/chunking/methods/subgraph_chunking/metadata.json +10 -0
- ragbench/chunking/methods/subgraph_chunking/strategy.py +6 -0
- ragbench/chunking/methods/table_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/table_based_chunking/metadata.json +14 -0
- ragbench/chunking/methods/table_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/table_level_chunking/__init__.py +3 -0
- ragbench/chunking/methods/table_level_chunking/metadata.json +14 -0
- ragbench/chunking/methods/table_level_chunking/strategy.py +6 -0
- ragbench/chunking/methods/table_region_chunking/__init__.py +3 -0
- ragbench/chunking/methods/table_region_chunking/metadata.json +12 -0
- ragbench/chunking/methods/table_region_chunking/strategy.py +6 -0
- ragbench/chunking/methods/thread_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/thread_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/thread_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/time_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/time_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/time_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/time_window_conversation_chunking/__init__.py +3 -0
- ragbench/chunking/methods/time_window_conversation_chunking/metadata.json +10 -0
- ragbench/chunking/methods/time_window_conversation_chunking/strategy.py +6 -0
- ragbench/chunking/methods/time_window_video_chunking/__init__.py +3 -0
- ragbench/chunking/methods/time_window_video_chunking/metadata.json +10 -0
- ragbench/chunking/methods/time_window_video_chunking/strategy.py +6 -0
- ragbench/chunking/methods/topic_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/topic_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/topic_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/transcript_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/transcript_based_chunking/metadata.json +10 -0
- ragbench/chunking/methods/transcript_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/transcript_based_video_chunking/__init__.py +3 -0
- ragbench/chunking/methods/transcript_based_video_chunking/metadata.json +10 -0
- ragbench/chunking/methods/transcript_based_video_chunking/strategy.py +6 -0
- ragbench/chunking/methods/vision_semantic_region_chunking/__init__.py +3 -0
- ragbench/chunking/methods/vision_semantic_region_chunking/metadata.json +16 -0
- ragbench/chunking/methods/vision_semantic_region_chunking/strategy.py +6 -0
- ragbench/chunking/methods/xml_tree_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/xml_tree_based_chunking/metadata.json +12 -0
- ragbench/chunking/methods/xml_tree_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/xpath_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/xpath_based_chunking/metadata.json +12 -0
- ragbench/chunking/methods/xpath_based_chunking/strategy.py +6 -0
- ragbench/chunking/methods/yaml_structure_based_chunking/__init__.py +3 -0
- ragbench/chunking/methods/yaml_structure_based_chunking/metadata.json +13 -0
- ragbench/chunking/methods/yaml_structure_based_chunking/strategy.py +6 -0
- ragbench/chunking/models.py +13 -0
- ragbench/chunking/registry.py +90 -0
- ragbench/cli.py +387 -0
- ragbench/connectors/__init__.py +12 -0
- ragbench/connectors/aac/connector.py +31 -0
- ragbench/connectors/aac/metadata.json +8 -0
- ragbench/connectors/aac/requirements.txt +1 -0
- ragbench/connectors/arangodb/connector.py +83 -0
- ragbench/connectors/arangodb/metadata.json +8 -0
- ragbench/connectors/arangodb/requirements.txt +1 -0
- ragbench/connectors/avi/connector.py +30 -0
- ragbench/connectors/avi/metadata.json +8 -0
- ragbench/connectors/avi/requirements.txt +1 -0
- ragbench/connectors/avro/connector.py +32 -0
- ragbench/connectors/avro/metadata.json +8 -0
- ragbench/connectors/avro/requirements.txt +1 -0
- ragbench/connectors/azure_blob/connector.py +200 -0
- ragbench/connectors/azure_blob/metadata.json +17 -0
- ragbench/connectors/azure_blob/requirements.txt +1 -0
- ragbench/connectors/bigquery/connector.py +91 -0
- ragbench/connectors/bigquery/metadata.json +8 -0
- ragbench/connectors/bigquery/requirements.txt +1 -0
- ragbench/connectors/bmp/connector.py +30 -0
- ragbench/connectors/bmp/metadata.json +8 -0
- ragbench/connectors/bmp/requirements.txt +1 -0
- ragbench/connectors/cassandra/connector.py +133 -0
- ragbench/connectors/cassandra/metadata.json +8 -0
- ragbench/connectors/cassandra/requirements.txt +1 -0
- ragbench/connectors/clickhouse/connector.py +112 -0
- ragbench/connectors/clickhouse/metadata.json +8 -0
- ragbench/connectors/clickhouse/requirements.txt +1 -0
- ragbench/connectors/csv/connector.py +30 -0
- ragbench/connectors/csv/metadata.json +8 -0
- ragbench/connectors/csv/requirements.txt +1 -0
- ragbench/connectors/databricks/connector.py +94 -0
- ragbench/connectors/databricks/metadata.json +8 -0
- ragbench/connectors/databricks/requirements.txt +1 -0
- ragbench/connectors/docx/connector.py +31 -0
- ragbench/connectors/docx/metadata.json +8 -0
- ragbench/connectors/docx/requirements.txt +1 -0
- ragbench/connectors/duckdb/connector.py +93 -0
- ragbench/connectors/duckdb/metadata.json +8 -0
- ragbench/connectors/duckdb/requirements.txt +1 -0
- ragbench/connectors/dynamodb/connector.py +79 -0
- ragbench/connectors/dynamodb/metadata.json +8 -0
- ragbench/connectors/dynamodb/requirements.txt +1 -0
- ragbench/connectors/elasticsearch/connector.py +92 -0
- ragbench/connectors/elasticsearch/metadata.json +8 -0
- ragbench/connectors/elasticsearch/requirements.txt +1 -0
- ragbench/connectors/excel/connector.py +30 -0
- ragbench/connectors/excel/metadata.json +9 -0
- ragbench/connectors/excel/requirements.txt +1 -0
- ragbench/connectors/feather/connector.py +30 -0
- ragbench/connectors/feather/metadata.json +8 -0
- ragbench/connectors/feather/requirements.txt +2 -0
- ragbench/connectors/flac/connector.py +31 -0
- ragbench/connectors/flac/metadata.json +8 -0
- ragbench/connectors/flac/requirements.txt +1 -0
- ragbench/connectors/gcs/connector.py +186 -0
- ragbench/connectors/gcs/metadata.json +15 -0
- ragbench/connectors/gcs/requirements.txt +2 -0
- ragbench/connectors/google_drive/connector.py +276 -0
- ragbench/connectors/google_drive/metadata.json +16 -0
- ragbench/connectors/google_drive/requirements.txt +3 -0
- ragbench/connectors/gzip/connector.py +30 -0
- ragbench/connectors/gzip/metadata.json +9 -0
- ragbench/connectors/gzip/requirements.txt +1 -0
- ragbench/connectors/html/connector.py +32 -0
- ragbench/connectors/html/metadata.json +9 -0
- ragbench/connectors/html/requirements.txt +1 -0
- ragbench/connectors/jira/connector.py +212 -0
- ragbench/connectors/jira/metadata.json +17 -0
- ragbench/connectors/jira/requirements.txt +1 -0
- ragbench/connectors/jpeg/connector.py +30 -0
- ragbench/connectors/jpeg/metadata.json +9 -0
- ragbench/connectors/jpeg/requirements.txt +1 -0
- ragbench/connectors/json/connector.py +35 -0
- ragbench/connectors/json/metadata.json +8 -0
- ragbench/connectors/json/requirements.txt +1 -0
- ragbench/connectors/jsonl/connector.py +30 -0
- ragbench/connectors/jsonl/metadata.json +8 -0
- ragbench/connectors/jsonl/requirements.txt +1 -0
- ragbench/connectors/kafka/connector.py +260 -0
- ragbench/connectors/kafka/metadata.json +17 -0
- ragbench/connectors/kafka/requirements.txt +1 -0
- ragbench/connectors/mariadb/connector.py +135 -0
- ragbench/connectors/mariadb/metadata.json +8 -0
- ragbench/connectors/mariadb/requirements.txt +2 -0
- ragbench/connectors/markdown/connector.py +32 -0
- ragbench/connectors/markdown/metadata.json +8 -0
- ragbench/connectors/markdown/requirements.txt +1 -0
- ragbench/connectors/memcached/connector.py +68 -0
- ragbench/connectors/memcached/metadata.json +8 -0
- ragbench/connectors/memcached/requirements.txt +1 -0
- ragbench/connectors/milvus/connector.py +80 -0
- ragbench/connectors/milvus/metadata.json +8 -0
- ragbench/connectors/milvus/requirements.txt +1 -0
- ragbench/connectors/minio/connector.py +188 -0
- ragbench/connectors/minio/metadata.json +18 -0
- ragbench/connectors/minio/requirements.txt +1 -0
- ragbench/connectors/mkv/connector.py +30 -0
- ragbench/connectors/mkv/metadata.json +8 -0
- ragbench/connectors/mkv/requirements.txt +1 -0
- ragbench/connectors/mongodb/connector.py +96 -0
- ragbench/connectors/mongodb/metadata.json +8 -0
- ragbench/connectors/mongodb/requirements.txt +1 -0
- ragbench/connectors/mov/connector.py +30 -0
- ragbench/connectors/mov/metadata.json +8 -0
- ragbench/connectors/mov/requirements.txt +1 -0
- ragbench/connectors/mp3/connector.py +31 -0
- ragbench/connectors/mp3/metadata.json +8 -0
- ragbench/connectors/mp3/requirements.txt +1 -0
- ragbench/connectors/mp4/connector.py +30 -0
- ragbench/connectors/mp4/metadata.json +8 -0
- ragbench/connectors/mp4/requirements.txt +1 -0
- ragbench/connectors/mssql/connector.py +134 -0
- ragbench/connectors/mssql/metadata.json +8 -0
- ragbench/connectors/mssql/requirements.txt +2 -0
- ragbench/connectors/mysql/connector.py +136 -0
- ragbench/connectors/mysql/metadata.json +8 -0
- ragbench/connectors/mysql/requirements.txt +2 -0
- ragbench/connectors/neo4j/connector.py +95 -0
- ragbench/connectors/neo4j/metadata.json +8 -0
- ragbench/connectors/neo4j/requirements.txt +1 -0
- ragbench/connectors/notion/connector.py +268 -0
- ragbench/connectors/notion/metadata.json +16 -0
- ragbench/connectors/notion/requirements.txt +1 -0
- ragbench/connectors/odp/connector.py +30 -0
- ragbench/connectors/odp/metadata.json +8 -0
- ragbench/connectors/odp/requirements.txt +1 -0
- ragbench/connectors/ods/connector.py +30 -0
- ragbench/connectors/ods/metadata.json +8 -0
- ragbench/connectors/ods/requirements.txt +2 -0
- ragbench/connectors/odt/connector.py +30 -0
- ragbench/connectors/odt/metadata.json +8 -0
- ragbench/connectors/odt/requirements.txt +1 -0
- ragbench/connectors/oracle/connector.py +139 -0
- ragbench/connectors/oracle/metadata.json +8 -0
- ragbench/connectors/oracle/requirements.txt +2 -0
- ragbench/connectors/orc/connector.py +31 -0
- ragbench/connectors/orc/metadata.json +8 -0
- ragbench/connectors/orc/requirements.txt +2 -0
- ragbench/connectors/parquet/connector.py +31 -0
- ragbench/connectors/parquet/metadata.json +8 -0
- ragbench/connectors/parquet/requirements.txt +3 -0
- ragbench/connectors/pdf_auto/connector.py +83 -0
- ragbench/connectors/pdf_auto/metadata.json +8 -0
- ragbench/connectors/pdf_auto/requirements.txt +4 -0
- ragbench/connectors/pdf_hybrid/connector.py +84 -0
- ragbench/connectors/pdf_hybrid/metadata.json +6 -0
- ragbench/connectors/pdf_hybrid/requirements.txt +4 -0
- ragbench/connectors/pdf_scanned/connector.py +56 -0
- ragbench/connectors/pdf_scanned/metadata.json +6 -0
- ragbench/connectors/pdf_scanned/requirements.txt +3 -0
- ragbench/connectors/pdf_text/connector.py +33 -0
- ragbench/connectors/pdf_text/metadata.json +6 -0
- ragbench/connectors/pdf_text/requirements.txt +1 -0
- ragbench/connectors/pinecone/connector.py +74 -0
- ragbench/connectors/pinecone/metadata.json +8 -0
- ragbench/connectors/pinecone/requirements.txt +1 -0
- ragbench/connectors/png/connector.py +30 -0
- ragbench/connectors/png/metadata.json +8 -0
- ragbench/connectors/png/requirements.txt +1 -0
- ragbench/connectors/postgresql/connector.py +161 -0
- ragbench/connectors/postgresql/metadata.json +8 -0
- ragbench/connectors/postgresql/requirements.txt +2 -0
- ragbench/connectors/pptx/connector.py +36 -0
- ragbench/connectors/pptx/metadata.json +8 -0
- ragbench/connectors/pptx/requirements.txt +1 -0
- ragbench/connectors/presto/connector.py +103 -0
- ragbench/connectors/presto/metadata.json +8 -0
- ragbench/connectors/presto/requirements.txt +1 -0
- ragbench/connectors/qdrant/connector.py +70 -0
- ragbench/connectors/qdrant/metadata.json +8 -0
- ragbench/connectors/qdrant/requirements.txt +1 -0
- ragbench/connectors/rabbitmq/connector.py +232 -0
- ragbench/connectors/rabbitmq/metadata.json +17 -0
- ragbench/connectors/rabbitmq/requirements.txt +1 -0
- ragbench/connectors/redis/connector.py +97 -0
- ragbench/connectors/redis/metadata.json +8 -0
- ragbench/connectors/redis/requirements.txt +1 -0
- ragbench/connectors/redshift/connector.py +112 -0
- ragbench/connectors/redshift/metadata.json +8 -0
- ragbench/connectors/redshift/requirements.txt +1 -0
- ragbench/connectors/rtf/connector.py +31 -0
- ragbench/connectors/rtf/metadata.json +8 -0
- ragbench/connectors/rtf/requirements.txt +1 -0
- ragbench/connectors/s3/connector.py +202 -0
- ragbench/connectors/s3/metadata.json +17 -0
- ragbench/connectors/s3/requirements.txt +1 -0
- ragbench/connectors/sevenz/connector.py +30 -0
- ragbench/connectors/sevenz/metadata.json +8 -0
- ragbench/connectors/sevenz/requirements.txt +1 -0
- ragbench/connectors/sitemap/connector.py +279 -0
- ragbench/connectors/sitemap/metadata.json +15 -0
- ragbench/connectors/sitemap/requirements.txt +3 -0
- ragbench/connectors/slack/connector.py +219 -0
- ragbench/connectors/slack/metadata.json +16 -0
- ragbench/connectors/slack/requirements.txt +1 -0
- ragbench/connectors/snowflake/connector.py +108 -0
- ragbench/connectors/snowflake/metadata.json +8 -0
- ragbench/connectors/snowflake/requirements.txt +1 -0
- ragbench/connectors/solr/connector.py +70 -0
- ragbench/connectors/solr/metadata.json +8 -0
- ragbench/connectors/solr/requirements.txt +1 -0
- ragbench/connectors/sqlite/connector.py +112 -0
- ragbench/connectors/sqlite/metadata.json +9 -0
- ragbench/connectors/sqlite/requirements.txt +1 -0
- ragbench/connectors/supabase/connector.py +88 -0
- ragbench/connectors/supabase/metadata.json +8 -0
- ragbench/connectors/supabase/requirements.txt +1 -0
- ragbench/connectors/tar/connector.py +30 -0
- ragbench/connectors/tar/metadata.json +8 -0
- ragbench/connectors/tar/requirements.txt +1 -0
- ragbench/connectors/tiff/connector.py +30 -0
- ragbench/connectors/tiff/metadata.json +9 -0
- ragbench/connectors/tiff/requirements.txt +1 -0
- ragbench/connectors/trino/connector.py +103 -0
- ragbench/connectors/trino/metadata.json +8 -0
- ragbench/connectors/trino/requirements.txt +1 -0
- ragbench/connectors/tsv/connector.py +30 -0
- ragbench/connectors/tsv/metadata.json +8 -0
- ragbench/connectors/tsv/requirements.txt +1 -0
- ragbench/connectors/txt/connector.py +30 -0
- ragbench/connectors/txt/metadata.json +8 -0
- ragbench/connectors/txt/requirements.txt +0 -0
- ragbench/connectors/wav/connector.py +31 -0
- ragbench/connectors/wav/metadata.json +8 -0
- ragbench/connectors/wav/requirements.txt +1 -0
- ragbench/connectors/weaviate/connector.py +77 -0
- ragbench/connectors/weaviate/metadata.json +8 -0
- ragbench/connectors/weaviate/requirements.txt +1 -0
- ragbench/connectors/web_scraper/connector.py +261 -0
- ragbench/connectors/web_scraper/metadata.json +16 -0
- ragbench/connectors/web_scraper/requirements.txt +3 -0
- ragbench/connectors/webp/connector.py +30 -0
- ragbench/connectors/webp/metadata.json +8 -0
- ragbench/connectors/webp/requirements.txt +1 -0
- ragbench/connectors/xlsx/connector.py +38 -0
- ragbench/connectors/xlsx/metadata.json +8 -0
- ragbench/connectors/xlsx/requirements.txt +1 -0
- ragbench/connectors/xml/connector.py +35 -0
- ragbench/connectors/xml/metadata.json +8 -0
- ragbench/connectors/xml/requirements.txt +1 -0
- ragbench/connectors/yaml/connector.py +31 -0
- ragbench/connectors/yaml/metadata.json +9 -0
- ragbench/connectors/yaml/requirements.txt +1 -0
- ragbench/connectors/zip/connector.py +30 -0
- ragbench/connectors/zip/metadata.json +8 -0
- ragbench/connectors/zip/requirements.txt +1 -0
- ragbench/core/__init__.py +13 -0
- ragbench/core/base_connector.py +32 -0
- ragbench/core/connectors/local_structured/avro_connector.py +32 -0
- ragbench/core/connectors/local_unstructured/pdf_hybrid_connector.py +31 -0
- ragbench/core/connectors/local_unstructured/tiff_connector.py +30 -0
- ragbench/core/db_connector.py +104 -0
- ragbench/core/document/__init__.py +4 -0
- ragbench/core/document/document_model.py +27 -0
- ragbench/core/loader.py +269 -0
- ragbench/core/plugin_manager.py +75 -0
- ragbench/core/registry.py +473 -0
- ragbench/core/utilities.py +9 -0
- ragbench/embedding/README.md +144 -0
- ragbench/embedding/__init__.py +21 -0
- ragbench/embedding/cache.py +72 -0
- ragbench/embedding/embedder.py +139 -0
- ragbench/embedding/env.py +78 -0
- ragbench/embedding/errors.py +47 -0
- ragbench/embedding/methods/__init__.py +4 -0
- ragbench/embedding/methods/_http.py +148 -0
- ragbench/embedding/methods/_local.py +81 -0
- ragbench/embedding/methods/_text.py +25 -0
- ragbench/embedding/methods/_wrapper.py +59 -0
- ragbench/embedding/methods/arctic/__init__.py +3 -0
- ragbench/embedding/methods/arctic/embedder.py +7 -0
- ragbench/embedding/methods/arctic/metadata.json +33 -0
- ragbench/embedding/methods/arctic/requirements.txt +1 -0
- ragbench/embedding/methods/audio_embedder/__init__.py +3 -0
- ragbench/embedding/methods/audio_embedder/embedder.py +38 -0
- ragbench/embedding/methods/audio_embedder/metadata.json +32 -0
- ragbench/embedding/methods/audio_embedder/requirements.txt +3 -0
- ragbench/embedding/methods/azure_openai/__init__.py +3 -0
- ragbench/embedding/methods/azure_openai/embedder.py +20 -0
- ragbench/embedding/methods/azure_openai/metadata.json +42 -0
- ragbench/embedding/methods/azure_openai/requirements.txt +0 -0
- ragbench/embedding/methods/base.py +303 -0
- ragbench/embedding/methods/bedrock/__init__.py +3 -0
- ragbench/embedding/methods/bedrock/embedder.py +51 -0
- ragbench/embedding/methods/bedrock/metadata.json +37 -0
- ragbench/embedding/methods/bedrock/requirements.txt +1 -0
- ragbench/embedding/methods/bge/__init__.py +3 -0
- ragbench/embedding/methods/bge/embedder.py +7 -0
- ragbench/embedding/methods/bge/metadata.json +33 -0
- ragbench/embedding/methods/bge/requirements.txt +1 -0
- ragbench/embedding/methods/bge_m3/__init__.py +3 -0
- ragbench/embedding/methods/bge_m3/embedder.py +34 -0
- ragbench/embedding/methods/bge_m3/metadata.json +39 -0
- ragbench/embedding/methods/bge_m3/requirements.txt +1 -0
- ragbench/embedding/methods/biomedical/__init__.py +3 -0
- ragbench/embedding/methods/biomedical/embedder.py +7 -0
- ragbench/embedding/methods/biomedical/metadata.json +33 -0
- ragbench/embedding/methods/biomedical/requirements.txt +1 -0
- ragbench/embedding/methods/bm25/__init__.py +3 -0
- ragbench/embedding/methods/bm25/embedder.py +45 -0
- ragbench/embedding/methods/bm25/metadata.json +26 -0
- ragbench/embedding/methods/bm25/requirements.txt +0 -0
- ragbench/embedding/methods/clap/__init__.py +3 -0
- ragbench/embedding/methods/clap/embedder.py +39 -0
- ragbench/embedding/methods/clap/metadata.json +29 -0
- ragbench/embedding/methods/clap/requirements.txt +3 -0
- ragbench/embedding/methods/clip/__init__.py +3 -0
- ragbench/embedding/methods/clip/embedder.py +34 -0
- ragbench/embedding/methods/clip/metadata.json +28 -0
- ragbench/embedding/methods/clip/requirements.txt +2 -0
- ragbench/embedding/methods/code_api/__init__.py +3 -0
- ragbench/embedding/methods/code_api/embedder.py +17 -0
- ragbench/embedding/methods/code_api/metadata.json +38 -0
- ragbench/embedding/methods/code_api/requirements.txt +0 -0
- ragbench/embedding/methods/code_local/__init__.py +3 -0
- ragbench/embedding/methods/code_local/embedder.py +15 -0
- ragbench/embedding/methods/code_local/metadata.json +33 -0
- ragbench/embedding/methods/code_local/requirements.txt +1 -0
- ragbench/embedding/methods/cohere/__init__.py +3 -0
- ragbench/embedding/methods/cohere/embedder.py +16 -0
- ragbench/embedding/methods/cohere/metadata.json +38 -0
- ragbench/embedding/methods/cohere/requirements.txt +0 -0
- ragbench/embedding/methods/colbert/__init__.py +3 -0
- ragbench/embedding/methods/colbert/embedder.py +25 -0
- ragbench/embedding/methods/colbert/metadata.json +23 -0
- ragbench/embedding/methods/colbert/requirements.txt +1 -0
- ragbench/embedding/methods/colpali/__init__.py +3 -0
- ragbench/embedding/methods/colpali/embedder.py +44 -0
- ragbench/embedding/methods/colpali/metadata.json +28 -0
- ragbench/embedding/methods/colpali/requirements.txt +3 -0
- ragbench/embedding/methods/contextual_prefix/__init__.py +3 -0
- ragbench/embedding/methods/contextual_prefix/embedder.py +55 -0
- ragbench/embedding/methods/contextual_prefix/metadata.json +36 -0
- ragbench/embedding/methods/contextual_prefix/requirements.txt +0 -0
- ragbench/embedding/methods/custom_callable/__init__.py +3 -0
- ragbench/embedding/methods/custom_callable/embedder.py +22 -0
- ragbench/embedding/methods/custom_callable/metadata.json +25 -0
- ragbench/embedding/methods/custom_callable/requirements.txt +0 -0
- ragbench/embedding/methods/custom_hf/__init__.py +3 -0
- ragbench/embedding/methods/custom_hf/embedder.py +11 -0
- ragbench/embedding/methods/custom_hf/metadata.json +37 -0
- ragbench/embedding/methods/custom_hf/requirements.txt +2 -0
- ragbench/embedding/methods/doc2vec/__init__.py +3 -0
- ragbench/embedding/methods/doc2vec/embedder.py +27 -0
- ragbench/embedding/methods/doc2vec/metadata.json +26 -0
- ragbench/embedding/methods/doc2vec/requirements.txt +1 -0
- ragbench/embedding/methods/e5/__init__.py +3 -0
- ragbench/embedding/methods/e5/embedder.py +7 -0
- ragbench/embedding/methods/e5/metadata.json +33 -0
- ragbench/embedding/methods/e5/requirements.txt +1 -0
- ragbench/embedding/methods/ensemble_concat/__init__.py +3 -0
- ragbench/embedding/methods/ensemble_concat/embedder.py +36 -0
- ragbench/embedding/methods/ensemble_concat/metadata.json +28 -0
- ragbench/embedding/methods/ensemble_concat/requirements.txt +0 -0
- ragbench/embedding/methods/fasttext/__init__.py +3 -0
- ragbench/embedding/methods/fasttext/embedder.py +26 -0
- ragbench/embedding/methods/fasttext/metadata.json +23 -0
- ragbench/embedding/methods/fasttext/requirements.txt +1 -0
- ragbench/embedding/methods/finance/__init__.py +3 -0
- ragbench/embedding/methods/finance/embedder.py +7 -0
- ragbench/embedding/methods/finance/metadata.json +33 -0
- ragbench/embedding/methods/finance/requirements.txt +1 -0
- ragbench/embedding/methods/fusion/__init__.py +3 -0
- ragbench/embedding/methods/fusion/embedder.py +26 -0
- ragbench/embedding/methods/fusion/metadata.json +28 -0
- ragbench/embedding/methods/fusion/requirements.txt +0 -0
- ragbench/embedding/methods/glove/__init__.py +3 -0
- ragbench/embedding/methods/glove/embedder.py +70 -0
- ragbench/embedding/methods/glove/metadata.json +23 -0
- ragbench/embedding/methods/glove/requirements.txt +0 -0
- ragbench/embedding/methods/google_gemini/__init__.py +3 -0
- ragbench/embedding/methods/google_gemini/embedder.py +26 -0
- ragbench/embedding/methods/google_gemini/metadata.json +42 -0
- ragbench/embedding/methods/google_gemini/requirements.txt +0 -0
- ragbench/embedding/methods/google_vertex/__init__.py +3 -0
- ragbench/embedding/methods/google_vertex/embedder.py +28 -0
- ragbench/embedding/methods/google_vertex/metadata.json +46 -0
- ragbench/embedding/methods/google_vertex/requirements.txt +0 -0
- ragbench/embedding/methods/gte/__init__.py +3 -0
- ragbench/embedding/methods/gte/embedder.py +7 -0
- ragbench/embedding/methods/gte/metadata.json +33 -0
- ragbench/embedding/methods/gte/requirements.txt +1 -0
- ragbench/embedding/methods/hash_baseline/__init__.py +3 -0
- ragbench/embedding/methods/hash_baseline/embedder.py +39 -0
- ragbench/embedding/methods/hash_baseline/metadata.json +16 -0
- ragbench/embedding/methods/hash_baseline/requirements.txt +0 -0
- ragbench/embedding/methods/hashing_sparse/__init__.py +3 -0
- ragbench/embedding/methods/hashing_sparse/embedder.py +24 -0
- ragbench/embedding/methods/hashing_sparse/metadata.json +26 -0
- ragbench/embedding/methods/hashing_sparse/requirements.txt +0 -0
- ragbench/embedding/methods/hashing_vectorizer/__init__.py +3 -0
- ragbench/embedding/methods/hashing_vectorizer/embedder.py +35 -0
- ragbench/embedding/methods/hashing_vectorizer/metadata.json +33 -0
- ragbench/embedding/methods/hashing_vectorizer/requirements.txt +0 -0
- ragbench/embedding/methods/hf_transformers/__init__.py +3 -0
- ragbench/embedding/methods/hf_transformers/embedder.py +5 -0
- ragbench/embedding/methods/hf_transformers/metadata.json +43 -0
- ragbench/embedding/methods/hf_transformers/requirements.txt +2 -0
- ragbench/embedding/methods/image_embedder/__init__.py +3 -0
- ragbench/embedding/methods/image_embedder/embedder.py +36 -0
- ragbench/embedding/methods/image_embedder/metadata.json +30 -0
- ragbench/embedding/methods/image_embedder/requirements.txt +3 -0
- ragbench/embedding/methods/jina/__init__.py +3 -0
- ragbench/embedding/methods/jina/embedder.py +14 -0
- ragbench/embedding/methods/jina/metadata.json +39 -0
- ragbench/embedding/methods/jina/requirements.txt +0 -0
- ragbench/embedding/methods/l2_normalize/__init__.py +3 -0
- ragbench/embedding/methods/l2_normalize/embedder.py +7 -0
- ragbench/embedding/methods/l2_normalize/metadata.json +27 -0
- ragbench/embedding/methods/l2_normalize/requirements.txt +0 -0
- ragbench/embedding/methods/late_chunking/__init__.py +3 -0
- ragbench/embedding/methods/late_chunking/embedder.py +89 -0
- ragbench/embedding/methods/late_chunking/metadata.json +31 -0
- ragbench/embedding/methods/late_chunking/requirements.txt +2 -0
- ragbench/embedding/methods/legal/__init__.py +3 -0
- ragbench/embedding/methods/legal/embedder.py +5 -0
- ragbench/embedding/methods/legal/metadata.json +39 -0
- ragbench/embedding/methods/legal/requirements.txt +2 -0
- ragbench/embedding/methods/llamacpp/__init__.py +3 -0
- ragbench/embedding/methods/llamacpp/embedder.py +11 -0
- ragbench/embedding/methods/llamacpp/metadata.json +39 -0
- ragbench/embedding/methods/llamacpp/requirements.txt +0 -0
- ragbench/embedding/methods/matryoshka/__init__.py +3 -0
- ragbench/embedding/methods/matryoshka/embedder.py +24 -0
- ragbench/embedding/methods/matryoshka/metadata.json +31 -0
- ragbench/embedding/methods/matryoshka/requirements.txt +0 -0
- ragbench/embedding/methods/minilm/__init__.py +3 -0
- ragbench/embedding/methods/minilm/embedder.py +7 -0
- ragbench/embedding/methods/minilm/metadata.json +33 -0
- ragbench/embedding/methods/minilm/requirements.txt +1 -0
- ragbench/embedding/methods/mistral/__init__.py +3 -0
- ragbench/embedding/methods/mistral/embedder.py +10 -0
- ragbench/embedding/methods/mistral/metadata.json +36 -0
- ragbench/embedding/methods/mistral/requirements.txt +0 -0
- ragbench/embedding/methods/multilingual_e5/__init__.py +3 -0
- ragbench/embedding/methods/multilingual_e5/embedder.py +7 -0
- ragbench/embedding/methods/multilingual_e5/metadata.json +35 -0
- ragbench/embedding/methods/multilingual_e5/requirements.txt +1 -0
- ragbench/embedding/methods/node2vec/__init__.py +3 -0
- ragbench/embedding/methods/node2vec/embedder.py +122 -0
- ragbench/embedding/methods/node2vec/metadata.json +45 -0
- ragbench/embedding/methods/node2vec/requirements.txt +1 -0
- ragbench/embedding/methods/nomic/__init__.py +3 -0
- ragbench/embedding/methods/nomic/embedder.py +11 -0
- ragbench/embedding/methods/nomic/metadata.json +33 -0
- ragbench/embedding/methods/nomic/requirements.txt +1 -0
- ragbench/embedding/methods/numeric_features/__init__.py +3 -0
- ragbench/embedding/methods/numeric_features/embedder.py +60 -0
- ragbench/embedding/methods/numeric_features/metadata.json +29 -0
- ragbench/embedding/methods/numeric_features/requirements.txt +1 -0
- ragbench/embedding/methods/ollama/__init__.py +3 -0
- ragbench/embedding/methods/ollama/embedder.py +20 -0
- ragbench/embedding/methods/ollama/metadata.json +35 -0
- ragbench/embedding/methods/ollama/requirements.txt +0 -0
- ragbench/embedding/methods/openai/__init__.py +3 -0
- ragbench/embedding/methods/openai/embedder.py +10 -0
- ragbench/embedding/methods/openai/metadata.json +36 -0
- ragbench/embedding/methods/openai/requirements.txt +0 -0
- ragbench/embedding/methods/openai_compatible/__init__.py +3 -0
- ragbench/embedding/methods/openai_compatible/embedder.py +10 -0
- ragbench/embedding/methods/openai_compatible/metadata.json +40 -0
- ragbench/embedding/methods/openai_compatible/requirements.txt +0 -0
- ragbench/embedding/methods/pca_reduce/__init__.py +3 -0
- ragbench/embedding/methods/pca_reduce/embedder.py +32 -0
- ragbench/embedding/methods/pca_reduce/metadata.json +33 -0
- ragbench/embedding/methods/pca_reduce/requirements.txt +0 -0
- ragbench/embedding/methods/quantize_binary/__init__.py +3 -0
- ragbench/embedding/methods/quantize_binary/embedder.py +17 -0
- ragbench/embedding/methods/quantize_binary/metadata.json +27 -0
- ragbench/embedding/methods/quantize_binary/requirements.txt +0 -0
- ragbench/embedding/methods/quantize_int8/__init__.py +3 -0
- ragbench/embedding/methods/quantize_int8/embedder.py +15 -0
- ragbench/embedding/methods/quantize_int8/metadata.json +26 -0
- ragbench/embedding/methods/quantize_int8/requirements.txt +0 -0
- ragbench/embedding/methods/sentence_transformers/__init__.py +1 -0
- ragbench/embedding/methods/sentence_transformers/embedder.py +96 -0
- ragbench/embedding/methods/sentence_transformers/metadata.json +20 -0
- ragbench/embedding/methods/sentence_transformers/requirements.txt +2 -0
- ragbench/embedding/methods/siglip/__init__.py +3 -0
- ragbench/embedding/methods/siglip/embedder.py +39 -0
- ragbench/embedding/methods/siglip/metadata.json +29 -0
- ragbench/embedding/methods/siglip/requirements.txt +3 -0
- ragbench/embedding/methods/splade/__init__.py +3 -0
- ragbench/embedding/methods/splade/embedder.py +38 -0
- ragbench/embedding/methods/splade/metadata.json +32 -0
- ragbench/embedding/methods/splade/requirements.txt +2 -0
- ragbench/embedding/methods/tabular_text/__init__.py +3 -0
- ragbench/embedding/methods/tabular_text/embedder.py +56 -0
- ragbench/embedding/methods/tabular_text/metadata.json +30 -0
- ragbench/embedding/methods/tabular_text/requirements.txt +0 -0
- ragbench/embedding/methods/tei/__init__.py +3 -0
- ragbench/embedding/methods/tei/embedder.py +18 -0
- ragbench/embedding/methods/tei/metadata.json +39 -0
- ragbench/embedding/methods/tei/requirements.txt +0 -0
- ragbench/embedding/methods/tfidf/__init__.py +3 -0
- ragbench/embedding/methods/tfidf/embedder.py +41 -0
- ragbench/embedding/methods/tfidf/metadata.json +23 -0
- ragbench/embedding/methods/tfidf/requirements.txt +0 -0
- ragbench/embedding/methods/transe/__init__.py +3 -0
- ragbench/embedding/methods/transe/embedder.py +111 -0
- ragbench/embedding/methods/transe/metadata.json +39 -0
- ragbench/embedding/methods/transe/requirements.txt +1 -0
- ragbench/embedding/methods/video_frames/__init__.py +3 -0
- ragbench/embedding/methods/video_frames/embedder.py +62 -0
- ragbench/embedding/methods/video_frames/metadata.json +30 -0
- ragbench/embedding/methods/video_frames/requirements.txt +1 -0
- ragbench/embedding/methods/vllm/__init__.py +3 -0
- ragbench/embedding/methods/vllm/embedder.py +10 -0
- ragbench/embedding/methods/vllm/metadata.json +37 -0
- ragbench/embedding/methods/vllm/requirements.txt +0 -0
- ragbench/embedding/methods/voyage/__init__.py +3 -0
- ragbench/embedding/methods/voyage/embedder.py +13 -0
- ragbench/embedding/methods/voyage/metadata.json +35 -0
- ragbench/embedding/methods/voyage/requirements.txt +0 -0
- ragbench/embedding/methods/word2vec/__init__.py +3 -0
- ragbench/embedding/methods/word2vec/embedder.py +70 -0
- ragbench/embedding/methods/word2vec/metadata.json +23 -0
- ragbench/embedding/methods/word2vec/requirements.txt +0 -0
- ragbench/embedding/models.py +84 -0
- ragbench/embedding/registry.py +139 -0
- ragbench/evaluation/BENCHMARKING.md +538 -0
- ragbench/evaluation/SWEEP_README.md +398 -0
- ragbench/evaluation/__init__.py +70 -0
- ragbench/evaluation/attribution.py +177 -0
- ragbench/evaluation/benchmark.py +476 -0
- ragbench/evaluation/ci_gate.py +164 -0
- ragbench/evaluation/evaluator.py +228 -0
- ragbench/evaluation/metrics/__init__.py +1 -0
- ragbench/evaluation/metrics/abstention/metadata.json +7 -0
- ragbench/evaluation/metrics/abstention/metric.py +110 -0
- ragbench/evaluation/metrics/answer_correctness/metadata.json +7 -0
- ragbench/evaluation/metrics/answer_correctness/metric.py +120 -0
- ragbench/evaluation/metrics/answer_relevancy/metadata.json +7 -0
- ragbench/evaluation/metrics/answer_relevancy/metric.py +116 -0
- ragbench/evaluation/metrics/base.py +138 -0
- ragbench/evaluation/metrics/citation_quality/metadata.json +7 -0
- ragbench/evaluation/metrics/citation_quality/metric.py +110 -0
- ragbench/evaluation/metrics/context_relevance/metadata.json +7 -0
- ragbench/evaluation/metrics/context_relevance/metric.py +122 -0
- ragbench/evaluation/metrics/faithfulness/metadata.json +7 -0
- ragbench/evaluation/metrics/faithfulness/metric.py +204 -0
- ragbench/evaluation/metrics/retrieval_ranking/metadata.json +7 -0
- ragbench/evaluation/metrics/retrieval_ranking/metric.py +165 -0
- ragbench/evaluation/metrics/system_metrics/metadata.json +7 -0
- ragbench/evaluation/metrics/system_metrics/metric.py +131 -0
- ragbench/evaluation/metrics/text_overlap/metadata.json +7 -0
- ragbench/evaluation/metrics/text_overlap/metric.py +311 -0
- ragbench/evaluation/models.py +238 -0
- ragbench/evaluation/registry.py +201 -0
- ragbench/evaluation/reporters.py +301 -0
- ragbench/evaluation/sweep.py +340 -0
- ragbench/evaluation/testset_generator.py +445 -0
- ragbench/generation/PROVIDERS.md +665 -0
- ragbench/generation/PROVIDERS_QUICK_REFERENCE.md +296 -0
- ragbench/generation/README.md +890 -0
- ragbench/generation/__init__.py +85 -0
- ragbench/generation/base.py +244 -0
- ragbench/generation/context_adapter.py +176 -0
- ragbench/generation/current_models.json +180 -0
- ragbench/generation/env.py +203 -0
- ragbench/generation/errors.py +49 -0
- ragbench/generation/generator.py +151 -0
- ragbench/generation/methods/anthropic/__init__.py +4 -0
- ragbench/generation/methods/anthropic/generator.py +288 -0
- ragbench/generation/methods/cohere/__init__.py +4 -0
- ragbench/generation/methods/cohere/generator.py +290 -0
- ragbench/generation/methods/gemini/__init__.py +4 -0
- ragbench/generation/methods/gemini/generator.py +284 -0
- ragbench/generation/methods/groq/__init__.py +4 -0
- ragbench/generation/methods/groq/generator.py +334 -0
- ragbench/generation/methods/mistral/__init__.py +4 -0
- ragbench/generation/methods/mistral/generator.py +333 -0
- ragbench/generation/methods/ollama/__init__.py +4 -0
- ragbench/generation/methods/ollama/generator.py +258 -0
- ragbench/generation/methods/openai/__init__.py +4 -0
- ragbench/generation/methods/openai/generator.py +302 -0
- ragbench/generation/methods/openrouter/__init__.py +4 -0
- ragbench/generation/methods/openrouter/generator.py +351 -0
- ragbench/generation/methods/together/__init__.py +4 -0
- ragbench/generation/methods/together/generator.py +334 -0
- ragbench/generation/model_config.py +209 -0
- ragbench/generation/model_validator.py +122 -0
- ragbench/generation/models.py +159 -0
- ragbench/generation/registry.py +101 -0
- ragbench/generation/requirements.txt +14 -0
- ragbench/ingestion/__init__.py +28 -0
- ragbench/pipeline/__init__.py +20 -0
- ragbench/pipeline/embedding_routing.py +88 -0
- ragbench/pipeline/pipeline.py +676 -0
- ragbench/pipeline/retrieval_routing.py +45 -0
- ragbench/pipeline/routing.py +70 -0
- ragbench/pipeline/vectorstore_routing.py +92 -0
- ragbench/reranking/__init__.py +36 -0
- ragbench/reranking/compression.py +61 -0
- ragbench/reranking/dedup.py +134 -0
- ragbench/reranking/env.py +80 -0
- ragbench/reranking/errors.py +43 -0
- ragbench/reranking/methods/__init__.py +4 -0
- ragbench/reranking/methods/_helpers.py +339 -0
- ragbench/reranking/methods/_http.py +180 -0
- ragbench/reranking/methods/auto/__init__.py +1 -0
- ragbench/reranking/methods/auto/metadata.json +14 -0
- ragbench/reranking/methods/auto/requirements.txt +1 -0
- ragbench/reranking/methods/auto/reranker.py +6 -0
- ragbench/reranking/methods/base.py +59 -0
- ragbench/reranking/methods/bge_reranker/__init__.py +0 -0
- ragbench/reranking/methods/bge_reranker/metadata.json +17 -0
- ragbench/reranking/methods/bge_reranker/requirements.txt +1 -0
- ragbench/reranking/methods/bge_reranker/reranker.py +5 -0
- ragbench/reranking/methods/bm25/__init__.py +1 -0
- ragbench/reranking/methods/bm25/metadata.json +14 -0
- ragbench/reranking/methods/bm25/requirements.txt +1 -0
- ragbench/reranking/methods/bm25/reranker.py +6 -0
- ragbench/reranking/methods/cohere/__init__.py +0 -0
- ragbench/reranking/methods/cohere/metadata.json +18 -0
- ragbench/reranking/methods/cohere/requirements.txt +0 -0
- ragbench/reranking/methods/cohere/reranker.py +4 -0
- ragbench/reranking/methods/colbert/__init__.py +0 -0
- ragbench/reranking/methods/colbert/metadata.json +15 -0
- ragbench/reranking/methods/colbert/requirements.txt +0 -0
- ragbench/reranking/methods/colbert/reranker.py +5 -0
- ragbench/reranking/methods/compression/__init__.py +1 -0
- ragbench/reranking/methods/compression/metadata.json +14 -0
- ragbench/reranking/methods/compression/requirements.txt +1 -0
- ragbench/reranking/methods/compression/reranker.py +6 -0
- ragbench/reranking/methods/custom_callable/__init__.py +1 -0
- ragbench/reranking/methods/custom_callable/metadata.json +14 -0
- ragbench/reranking/methods/custom_callable/requirements.txt +1 -0
- ragbench/reranking/methods/custom_callable/reranker.py +6 -0
- ragbench/reranking/methods/dedup/__init__.py +1 -0
- ragbench/reranking/methods/dedup/metadata.json +14 -0
- ragbench/reranking/methods/dedup/requirements.txt +1 -0
- ragbench/reranking/methods/dedup/reranker.py +6 -0
- ragbench/reranking/methods/diversity/__init__.py +1 -0
- ragbench/reranking/methods/diversity/metadata.json +14 -0
- ragbench/reranking/methods/diversity/requirements.txt +1 -0
- ragbench/reranking/methods/diversity/reranker.py +6 -0
- ragbench/reranking/methods/fusion/__init__.py +1 -0
- ragbench/reranking/methods/fusion/metadata.json +14 -0
- ragbench/reranking/methods/fusion/requirements.txt +1 -0
- ragbench/reranking/methods/fusion/reranker.py +6 -0
- ragbench/reranking/methods/jina/__init__.py +0 -0
- ragbench/reranking/methods/jina/metadata.json +18 -0
- ragbench/reranking/methods/jina/requirements.txt +0 -0
- ragbench/reranking/methods/jina/reranker.py +4 -0
- ragbench/reranking/methods/lexical/__init__.py +1 -0
- ragbench/reranking/methods/lexical/metadata.json +14 -0
- ragbench/reranking/methods/lexical/requirements.txt +1 -0
- ragbench/reranking/methods/lexical/reranker.py +6 -0
- ragbench/reranking/methods/llm_listwise/__init__.py +1 -0
- ragbench/reranking/methods/llm_listwise/metadata.json +14 -0
- ragbench/reranking/methods/llm_listwise/requirements.txt +1 -0
- ragbench/reranking/methods/llm_listwise/reranker.py +6 -0
- ragbench/reranking/methods/llm_pairwise/__init__.py +1 -0
- ragbench/reranking/methods/llm_pairwise/metadata.json +14 -0
- ragbench/reranking/methods/llm_pairwise/requirements.txt +1 -0
- ragbench/reranking/methods/llm_pairwise/reranker.py +6 -0
- ragbench/reranking/methods/metadata_boost/__init__.py +1 -0
- ragbench/reranking/methods/metadata_boost/metadata.json +14 -0
- ragbench/reranking/methods/metadata_boost/requirements.txt +1 -0
- ragbench/reranking/methods/metadata_boost/reranker.py +6 -0
- ragbench/reranking/methods/minilm/__init__.py +0 -0
- ragbench/reranking/methods/minilm/metadata.json +17 -0
- ragbench/reranking/methods/minilm/requirements.txt +1 -0
- ragbench/reranking/methods/minilm/reranker.py +5 -0
- ragbench/reranking/methods/noop/__init__.py +1 -0
- ragbench/reranking/methods/noop/metadata.json +14 -0
- ragbench/reranking/methods/noop/requirements.txt +1 -0
- ragbench/reranking/methods/noop/reranker.py +6 -0
- ragbench/reranking/methods/normalize/__init__.py +1 -0
- ragbench/reranking/methods/normalize/metadata.json +14 -0
- ragbench/reranking/methods/normalize/requirements.txt +1 -0
- ragbench/reranking/methods/normalize/reranker.py +6 -0
- ragbench/reranking/methods/threshold/__init__.py +1 -0
- ragbench/reranking/methods/threshold/metadata.json +14 -0
- ragbench/reranking/methods/threshold/requirements.txt +1 -0
- ragbench/reranking/methods/threshold/reranker.py +6 -0
- ragbench/reranking/methods/voyage/__init__.py +0 -0
- ragbench/reranking/methods/voyage/metadata.json +18 -0
- ragbench/reranking/methods/voyage/requirements.txt +0 -0
- ragbench/reranking/methods/voyage/reranker.py +4 -0
- ragbench/reranking/methods/window_compression/__init__.py +1 -0
- ragbench/reranking/methods/window_compression/metadata.json +14 -0
- ragbench/reranking/methods/window_compression/requirements.txt +1 -0
- ragbench/reranking/methods/window_compression/reranker.py +6 -0
- ragbench/reranking/models.py +157 -0
- ragbench/reranking/registry.py +161 -0
- ragbench/reranking/reranker.py +289 -0
- ragbench/reranking/routing.py +158 -0
- ragbench/reranking/text.py +124 -0
- ragbench/retrieval/README.md +83 -0
- ragbench/retrieval/__init__.py +30 -0
- ragbench/retrieval/diversity.py +111 -0
- ragbench/retrieval/env.py +80 -0
- ragbench/retrieval/errors.py +55 -0
- ragbench/retrieval/expansion.py +149 -0
- ragbench/retrieval/filters.py +107 -0
- ragbench/retrieval/fusion.py +152 -0
- ragbench/retrieval/methods/__init__.py +1 -0
- ragbench/retrieval/methods/_helpers.py +309 -0
- ragbench/retrieval/methods/auto/__init__.py +1 -0
- ragbench/retrieval/methods/auto/metadata.json +27 -0
- ragbench/retrieval/methods/auto/requirements.txt +0 -0
- ragbench/retrieval/methods/auto/retriever.py +6 -0
- ragbench/retrieval/methods/base.py +69 -0
- ragbench/retrieval/methods/bm25/__init__.py +1 -0
- ragbench/retrieval/methods/bm25/metadata.json +23 -0
- ragbench/retrieval/methods/bm25/requirements.txt +0 -0
- ragbench/retrieval/methods/bm25/retriever.py +6 -0
- ragbench/retrieval/methods/custom_callable/__init__.py +1 -0
- ragbench/retrieval/methods/custom_callable/metadata.json +24 -0
- ragbench/retrieval/methods/custom_callable/requirements.txt +0 -0
- ragbench/retrieval/methods/custom_callable/retriever.py +6 -0
- ragbench/retrieval/methods/decomposition/__init__.py +1 -0
- ragbench/retrieval/methods/decomposition/metadata.json +28 -0
- ragbench/retrieval/methods/decomposition/requirements.txt +0 -0
- ragbench/retrieval/methods/decomposition/retriever.py +6 -0
- ragbench/retrieval/methods/dense/__init__.py +1 -0
- ragbench/retrieval/methods/dense/metadata.json +24 -0
- ragbench/retrieval/methods/dense/requirements.txt +0 -0
- ragbench/retrieval/methods/dense/retriever.py +6 -0
- ragbench/retrieval/methods/dense_filtered/__init__.py +1 -0
- ragbench/retrieval/methods/dense_filtered/metadata.json +24 -0
- ragbench/retrieval/methods/dense_filtered/requirements.txt +0 -0
- ragbench/retrieval/methods/dense_filtered/retriever.py +6 -0
- ragbench/retrieval/methods/diversity_rerank/__init__.py +1 -0
- ragbench/retrieval/methods/diversity_rerank/metadata.json +26 -0
- ragbench/retrieval/methods/diversity_rerank/requirements.txt +0 -0
- ragbench/retrieval/methods/diversity_rerank/retriever.py +6 -0
- ragbench/retrieval/methods/filtered/__init__.py +1 -0
- ragbench/retrieval/methods/filtered/metadata.json +30 -0
- ragbench/retrieval/methods/filtered/requirements.txt +0 -0
- ragbench/retrieval/methods/filtered/retriever.py +6 -0
- ragbench/retrieval/methods/hash_baseline/__init__.py +1 -0
- ragbench/retrieval/methods/hash_baseline/metadata.json +22 -0
- ragbench/retrieval/methods/hash_baseline/requirements.txt +0 -0
- ragbench/retrieval/methods/hash_baseline/retriever.py +6 -0
- ragbench/retrieval/methods/hybrid_rerank/__init__.py +1 -0
- ragbench/retrieval/methods/hybrid_rerank/metadata.json +24 -0
- ragbench/retrieval/methods/hybrid_rerank/requirements.txt +0 -0
- ragbench/retrieval/methods/hybrid_rerank/retriever.py +6 -0
- ragbench/retrieval/methods/hybrid_rrf/__init__.py +1 -0
- ragbench/retrieval/methods/hybrid_rrf/metadata.json +29 -0
- ragbench/retrieval/methods/hybrid_rrf/requirements.txt +0 -0
- ragbench/retrieval/methods/hybrid_rrf/retriever.py +6 -0
- ragbench/retrieval/methods/hybrid_weighted/__init__.py +1 -0
- ragbench/retrieval/methods/hybrid_weighted/metadata.json +28 -0
- ragbench/retrieval/methods/hybrid_weighted/requirements.txt +0 -0
- ragbench/retrieval/methods/hybrid_weighted/retriever.py +6 -0
- ragbench/retrieval/methods/hyde/__init__.py +1 -0
- ragbench/retrieval/methods/hyde/metadata.json +26 -0
- ragbench/retrieval/methods/hyde/requirements.txt +0 -0
- ragbench/retrieval/methods/hyde/retriever.py +6 -0
- ragbench/retrieval/methods/mmr/__init__.py +1 -0
- ragbench/retrieval/methods/mmr/metadata.json +26 -0
- ragbench/retrieval/methods/mmr/requirements.txt +0 -0
- ragbench/retrieval/methods/mmr/retriever.py +6 -0
- ragbench/retrieval/methods/multi_collection/__init__.py +1 -0
- ragbench/retrieval/methods/multi_collection/metadata.json +27 -0
- ragbench/retrieval/methods/multi_collection/requirements.txt +0 -0
- ragbench/retrieval/methods/multi_collection/retriever.py +6 -0
- ragbench/retrieval/methods/multi_collection_rrf/__init__.py +1 -0
- ragbench/retrieval/methods/multi_collection_rrf/metadata.json +27 -0
- ragbench/retrieval/methods/multi_collection_rrf/requirements.txt +0 -0
- ragbench/retrieval/methods/multi_collection_rrf/retriever.py +6 -0
- ragbench/retrieval/methods/multi_query/__init__.py +1 -0
- ragbench/retrieval/methods/multi_query/metadata.json +30 -0
- ragbench/retrieval/methods/multi_query/requirements.txt +0 -0
- ragbench/retrieval/methods/multi_query/retriever.py +6 -0
- ragbench/retrieval/methods/multi_vector/__init__.py +1 -0
- ragbench/retrieval/methods/multi_vector/metadata.json +24 -0
- ragbench/retrieval/methods/multi_vector/requirements.txt +0 -0
- ragbench/retrieval/methods/multi_vector/retriever.py +6 -0
- ragbench/retrieval/methods/parent_child/__init__.py +1 -0
- ragbench/retrieval/methods/parent_child/metadata.json +30 -0
- ragbench/retrieval/methods/parent_child/requirements.txt +0 -0
- ragbench/retrieval/methods/parent_child/retriever.py +6 -0
- ragbench/retrieval/methods/query_rewrite/__init__.py +1 -0
- ragbench/retrieval/methods/query_rewrite/metadata.json +30 -0
- ragbench/retrieval/methods/query_rewrite/requirements.txt +0 -0
- ragbench/retrieval/methods/query_rewrite/retriever.py +6 -0
- ragbench/retrieval/methods/routed/__init__.py +1 -0
- ragbench/retrieval/methods/routed/metadata.json +27 -0
- ragbench/retrieval/methods/routed/requirements.txt +0 -0
- ragbench/retrieval/methods/routed/retriever.py +6 -0
- ragbench/retrieval/methods/rrf_fusion/__init__.py +1 -0
- ragbench/retrieval/methods/rrf_fusion/metadata.json +29 -0
- ragbench/retrieval/methods/rrf_fusion/requirements.txt +0 -0
- ragbench/retrieval/methods/rrf_fusion/retriever.py +6 -0
- ragbench/retrieval/methods/self_query/__init__.py +1 -0
- ragbench/retrieval/methods/self_query/metadata.json +26 -0
- ragbench/retrieval/methods/self_query/requirements.txt +0 -0
- ragbench/retrieval/methods/self_query/retriever.py +6 -0
- ragbench/retrieval/methods/sentence_window/__init__.py +1 -0
- ragbench/retrieval/methods/sentence_window/metadata.json +33 -0
- ragbench/retrieval/methods/sentence_window/requirements.txt +0 -0
- ragbench/retrieval/methods/sentence_window/retriever.py +6 -0
- ragbench/retrieval/methods/similarity_threshold/__init__.py +1 -0
- ragbench/retrieval/methods/similarity_threshold/metadata.json +30 -0
- ragbench/retrieval/methods/similarity_threshold/requirements.txt +0 -0
- ragbench/retrieval/methods/similarity_threshold/retriever.py +6 -0
- ragbench/retrieval/methods/sparse/__init__.py +1 -0
- ragbench/retrieval/methods/sparse/metadata.json +23 -0
- ragbench/retrieval/methods/sparse/requirements.txt +0 -0
- ragbench/retrieval/methods/sparse/retriever.py +6 -0
- ragbench/retrieval/methods/step_back/__init__.py +1 -0
- ragbench/retrieval/methods/step_back/metadata.json +26 -0
- ragbench/retrieval/methods/step_back/requirements.txt +0 -0
- ragbench/retrieval/methods/step_back/retriever.py +6 -0
- ragbench/retrieval/methods/weighted_fusion/__init__.py +1 -0
- ragbench/retrieval/methods/weighted_fusion/metadata.json +26 -0
- ragbench/retrieval/methods/weighted_fusion/requirements.txt +0 -0
- ragbench/retrieval/methods/weighted_fusion/retriever.py +6 -0
- ragbench/retrieval/models.py +168 -0
- ragbench/retrieval/query.py +160 -0
- ragbench/retrieval/registry.py +123 -0
- ragbench/retrieval/retriever.py +347 -0
- ragbench/retrieval/routing.py +130 -0
- ragbench/retrieval/self_query.py +62 -0
- ragbench/vectorstore/README.md +374 -0
- ragbench/vectorstore/__init__.py +32 -0
- ragbench/vectorstore/env.py +78 -0
- ragbench/vectorstore/errors.py +59 -0
- ragbench/vectorstore/filters.py +167 -0
- ragbench/vectorstore/methods/__init__.py +4 -0
- ragbench/vectorstore/methods/_ann.py +99 -0
- ragbench/vectorstore/methods/_bruteforce.py +170 -0
- ragbench/vectorstore/methods/_elastic.py +166 -0
- ragbench/vectorstore/methods/_embedded.py +49 -0
- ragbench/vectorstore/methods/_http.py +141 -0
- ragbench/vectorstore/methods/_qdrant_common.py +53 -0
- ragbench/vectorstore/methods/_scan.py +57 -0
- ragbench/vectorstore/methods/_sql.py +205 -0
- ragbench/vectorstore/methods/_wrapper.py +66 -0
- ragbench/vectorstore/methods/annoy/__init__.py +3 -0
- ragbench/vectorstore/methods/annoy/metadata.json +39 -0
- ragbench/vectorstore/methods/annoy/requirements.txt +2 -0
- ragbench/vectorstore/methods/annoy/store.py +25 -0
- ragbench/vectorstore/methods/astra_db/__init__.py +3 -0
- ragbench/vectorstore/methods/astra_db/metadata.json +61 -0
- ragbench/vectorstore/methods/astra_db/requirements.txt +0 -0
- ragbench/vectorstore/methods/astra_db/store.py +100 -0
- ragbench/vectorstore/methods/azure_ai_search/__init__.py +3 -0
- ragbench/vectorstore/methods/azure_ai_search/metadata.json +59 -0
- ragbench/vectorstore/methods/azure_ai_search/requirements.txt +0 -0
- ragbench/vectorstore/methods/azure_ai_search/store.py +83 -0
- ragbench/vectorstore/methods/base.py +607 -0
- ragbench/vectorstore/methods/bm25_local/__init__.py +3 -0
- ragbench/vectorstore/methods/bm25_local/metadata.json +43 -0
- ragbench/vectorstore/methods/bm25_local/requirements.txt +0 -0
- ragbench/vectorstore/methods/bm25_local/store.py +133 -0
- ragbench/vectorstore/methods/chroma/__init__.py +3 -0
- ragbench/vectorstore/methods/chroma/metadata.json +40 -0
- ragbench/vectorstore/methods/chroma/requirements.txt +1 -0
- ragbench/vectorstore/methods/chroma/store.py +120 -0
- ragbench/vectorstore/methods/clickhouse/__init__.py +3 -0
- ragbench/vectorstore/methods/clickhouse/metadata.json +64 -0
- ragbench/vectorstore/methods/clickhouse/requirements.txt +1 -0
- ragbench/vectorstore/methods/clickhouse/store.py +142 -0
- ragbench/vectorstore/methods/custom_callable/__init__.py +3 -0
- ragbench/vectorstore/methods/custom_callable/metadata.json +75 -0
- ragbench/vectorstore/methods/custom_callable/requirements.txt +0 -0
- ragbench/vectorstore/methods/custom_callable/store.py +40 -0
- ragbench/vectorstore/methods/dedup/__init__.py +3 -0
- ragbench/vectorstore/methods/dedup/metadata.json +54 -0
- ragbench/vectorstore/methods/dedup/requirements.txt +0 -0
- ragbench/vectorstore/methods/dedup/store.py +38 -0
- ragbench/vectorstore/methods/duckdb/__init__.py +3 -0
- ragbench/vectorstore/methods/duckdb/metadata.json +40 -0
- ragbench/vectorstore/methods/duckdb/requirements.txt +1 -0
- ragbench/vectorstore/methods/duckdb/store.py +116 -0
- ragbench/vectorstore/methods/elasticsearch/__init__.py +3 -0
- ragbench/vectorstore/methods/elasticsearch/metadata.json +58 -0
- ragbench/vectorstore/methods/elasticsearch/requirements.txt +0 -0
- ragbench/vectorstore/methods/elasticsearch/store.py +23 -0
- ragbench/vectorstore/methods/faiss/__init__.py +3 -0
- ragbench/vectorstore/methods/faiss/metadata.json +45 -0
- ragbench/vectorstore/methods/faiss/requirements.txt +2 -0
- ragbench/vectorstore/methods/faiss/store.py +52 -0
- ragbench/vectorstore/methods/flat_file/__init__.py +3 -0
- ragbench/vectorstore/methods/flat_file/metadata.json +40 -0
- ragbench/vectorstore/methods/flat_file/requirements.txt +0 -0
- ragbench/vectorstore/methods/flat_file/store.py +108 -0
- ragbench/vectorstore/methods/hnswlib/__init__.py +3 -0
- ragbench/vectorstore/methods/hnswlib/metadata.json +39 -0
- ragbench/vectorstore/methods/hnswlib/requirements.txt +2 -0
- ragbench/vectorstore/methods/hnswlib/store.py +28 -0
- ragbench/vectorstore/methods/lancedb/__init__.py +3 -0
- ragbench/vectorstore/methods/lancedb/metadata.json +41 -0
- ragbench/vectorstore/methods/lancedb/requirements.txt +2 -0
- ragbench/vectorstore/methods/lancedb/store.py +115 -0
- ragbench/vectorstore/methods/mariadb/__init__.py +3 -0
- ragbench/vectorstore/methods/mariadb/metadata.json +64 -0
- ragbench/vectorstore/methods/mariadb/requirements.txt +1 -0
- ragbench/vectorstore/methods/mariadb/store.py +39 -0
- ragbench/vectorstore/methods/meilisearch/__init__.py +3 -0
- ragbench/vectorstore/methods/meilisearch/metadata.json +55 -0
- ragbench/vectorstore/methods/meilisearch/requirements.txt +0 -0
- ragbench/vectorstore/methods/meilisearch/store.py +83 -0
- ragbench/vectorstore/methods/memory/__init__.py +3 -0
- ragbench/vectorstore/methods/memory/metadata.json +33 -0
- ragbench/vectorstore/methods/memory/requirements.txt +0 -0
- ragbench/vectorstore/methods/memory/store.py +5 -0
- ragbench/vectorstore/methods/milvus/__init__.py +3 -0
- ragbench/vectorstore/methods/milvus/metadata.json +60 -0
- ragbench/vectorstore/methods/milvus/requirements.txt +0 -0
- ragbench/vectorstore/methods/milvus/store.py +91 -0
- ragbench/vectorstore/methods/milvus_lite/__init__.py +3 -0
- ragbench/vectorstore/methods/milvus_lite/metadata.json +42 -0
- ragbench/vectorstore/methods/milvus_lite/requirements.txt +2 -0
- ragbench/vectorstore/methods/milvus_lite/store.py +124 -0
- ragbench/vectorstore/methods/mirror/__init__.py +3 -0
- ragbench/vectorstore/methods/mirror/metadata.json +59 -0
- ragbench/vectorstore/methods/mirror/requirements.txt +0 -0
- ragbench/vectorstore/methods/mirror/store.py +49 -0
- ragbench/vectorstore/methods/mongodb_atlas/__init__.py +3 -0
- ragbench/vectorstore/methods/mongodb_atlas/metadata.json +59 -0
- ragbench/vectorstore/methods/mongodb_atlas/requirements.txt +1 -0
- ragbench/vectorstore/methods/mongodb_atlas/store.py +92 -0
- ragbench/vectorstore/methods/namespace/__init__.py +3 -0
- ragbench/vectorstore/methods/namespace/metadata.json +58 -0
- ragbench/vectorstore/methods/namespace/requirements.txt +0 -0
- ragbench/vectorstore/methods/namespace/store.py +51 -0
- ragbench/vectorstore/methods/neo4j/__init__.py +3 -0
- ragbench/vectorstore/methods/neo4j/metadata.json +64 -0
- ragbench/vectorstore/methods/neo4j/requirements.txt +0 -0
- ragbench/vectorstore/methods/neo4j/store.py +97 -0
- ragbench/vectorstore/methods/numpy_flat/__init__.py +3 -0
- ragbench/vectorstore/methods/numpy_flat/metadata.json +40 -0
- ragbench/vectorstore/methods/numpy_flat/requirements.txt +1 -0
- ragbench/vectorstore/methods/numpy_flat/store.py +113 -0
- ragbench/vectorstore/methods/opensearch/__init__.py +3 -0
- ragbench/vectorstore/methods/opensearch/metadata.json +64 -0
- ragbench/vectorstore/methods/opensearch/requirements.txt +0 -0
- ragbench/vectorstore/methods/opensearch/store.py +40 -0
- ragbench/vectorstore/methods/oracle_23ai/__init__.py +3 -0
- ragbench/vectorstore/methods/oracle_23ai/metadata.json +62 -0
- ragbench/vectorstore/methods/oracle_23ai/requirements.txt +1 -0
- ragbench/vectorstore/methods/oracle_23ai/store.py +48 -0
- ragbench/vectorstore/methods/pgvector/__init__.py +3 -0
- ragbench/vectorstore/methods/pgvector/metadata.json +58 -0
- ragbench/vectorstore/methods/pgvector/requirements.txt +1 -0
- ragbench/vectorstore/methods/pgvector/store.py +133 -0
- ragbench/vectorstore/methods/pinecone/__init__.py +3 -0
- ragbench/vectorstore/methods/pinecone/metadata.json +59 -0
- ragbench/vectorstore/methods/pinecone/requirements.txt +0 -0
- ragbench/vectorstore/methods/pinecone/store.py +148 -0
- ragbench/vectorstore/methods/qdrant/__init__.py +3 -0
- ragbench/vectorstore/methods/qdrant/metadata.json +61 -0
- ragbench/vectorstore/methods/qdrant/requirements.txt +0 -0
- ragbench/vectorstore/methods/qdrant/store.py +127 -0
- ragbench/vectorstore/methods/qdrant_local/__init__.py +3 -0
- ragbench/vectorstore/methods/qdrant_local/metadata.json +58 -0
- ragbench/vectorstore/methods/qdrant_local/requirements.txt +1 -0
- ragbench/vectorstore/methods/qdrant_local/store.py +147 -0
- ragbench/vectorstore/methods/read_cache/__init__.py +3 -0
- ragbench/vectorstore/methods/read_cache/metadata.json +57 -0
- ragbench/vectorstore/methods/read_cache/requirements.txt +0 -0
- ragbench/vectorstore/methods/read_cache/store.py +36 -0
- ragbench/vectorstore/methods/redis/__init__.py +3 -0
- ragbench/vectorstore/methods/redis/metadata.json +57 -0
- ragbench/vectorstore/methods/redis/requirements.txt +1 -0
- ragbench/vectorstore/methods/redis/store.py +156 -0
- ragbench/vectorstore/methods/shard_router/__init__.py +3 -0
- ragbench/vectorstore/methods/shard_router/metadata.json +56 -0
- ragbench/vectorstore/methods/shard_router/requirements.txt +0 -0
- ragbench/vectorstore/methods/shard_router/store.py +101 -0
- ragbench/vectorstore/methods/singlestore/__init__.py +3 -0
- ragbench/vectorstore/methods/singlestore/metadata.json +53 -0
- ragbench/vectorstore/methods/singlestore/requirements.txt +1 -0
- ragbench/vectorstore/methods/singlestore/store.py +39 -0
- ragbench/vectorstore/methods/sqlite/__init__.py +3 -0
- ragbench/vectorstore/methods/sqlite/metadata.json +39 -0
- ragbench/vectorstore/methods/sqlite/requirements.txt +0 -0
- ragbench/vectorstore/methods/sqlite/store.py +150 -0
- ragbench/vectorstore/methods/sqlite_vec/__init__.py +3 -0
- ragbench/vectorstore/methods/sqlite_vec/metadata.json +40 -0
- ragbench/vectorstore/methods/sqlite_vec/requirements.txt +1 -0
- ragbench/vectorstore/methods/sqlite_vec/store.py +58 -0
- ragbench/vectorstore/methods/turbopuffer/__init__.py +3 -0
- ragbench/vectorstore/methods/turbopuffer/metadata.json +56 -0
- ragbench/vectorstore/methods/turbopuffer/requirements.txt +0 -0
- ragbench/vectorstore/methods/turbopuffer/store.py +65 -0
- ragbench/vectorstore/methods/typesense/__init__.py +3 -0
- ragbench/vectorstore/methods/typesense/metadata.json +55 -0
- ragbench/vectorstore/methods/typesense/requirements.txt +0 -0
- ragbench/vectorstore/methods/typesense/store.py +82 -0
- ragbench/vectorstore/methods/upstash/__init__.py +3 -0
- ragbench/vectorstore/methods/upstash/metadata.json +57 -0
- ragbench/vectorstore/methods/upstash/requirements.txt +0 -0
- ragbench/vectorstore/methods/upstash/store.py +52 -0
- ragbench/vectorstore/methods/usearch/__init__.py +3 -0
- ragbench/vectorstore/methods/usearch/metadata.json +39 -0
- ragbench/vectorstore/methods/usearch/requirements.txt +2 -0
- ragbench/vectorstore/methods/usearch/store.py +29 -0
- ragbench/vectorstore/methods/weaviate/__init__.py +3 -0
- ragbench/vectorstore/methods/weaviate/metadata.json +58 -0
- ragbench/vectorstore/methods/weaviate/requirements.txt +0 -0
- ragbench/vectorstore/methods/weaviate/store.py +99 -0
- ragbench/vectorstore/models.py +135 -0
- ragbench/vectorstore/registry.py +115 -0
- ragbench/vectorstore/store.py +97 -0
- ragbench_ai-0.4.0.dist-info/METADATA +546 -0
- ragbench_ai-0.4.0.dist-info/RECORD +1336 -0
- ragbench_ai-0.4.0.dist-info/WHEEL +5 -0
- ragbench_ai-0.4.0.dist-info/entry_points.txt +2 -0
- ragbench_ai-0.4.0.dist-info/licenses/LICENSE +201 -0
- ragbench_ai-0.4.0.dist-info/top_level.txt +1 -0
ragbench/__init__.py
ADDED
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
"""RAGBench: Layered RAG framework with independent, auto-chainable components.
|
|
2
|
+
|
|
3
|
+
Usage:
|
|
4
|
+
from ragbench import OmniRAG, ask
|
|
5
|
+
from ragbench.pipeline import auto
|
|
6
|
+
from ragbench.embedding import embed
|
|
7
|
+
from ragbench.retrieval import retrieve
|
|
8
|
+
from ragbench.generation import Generator
|
|
9
|
+
from ragbench.evaluation import Evaluator
|
|
10
|
+
"""
|
|
11
|
+
|
|
12
|
+
__version__ = "0.4.0"
|
|
13
|
+
|
|
14
|
+
# Export main pipeline functions
|
|
15
|
+
from ragbench.pipeline import ask, auto, OmniPipeline
|
|
16
|
+
|
|
17
|
+
# Alias OmniPipeline as OmniRAG for convenience
|
|
18
|
+
OmniRAG = OmniPipeline
|
|
19
|
+
|
|
20
|
+
__all__ = [
|
|
21
|
+
"__version__",
|
|
22
|
+
"ask",
|
|
23
|
+
"auto",
|
|
24
|
+
"OmniPipeline",
|
|
25
|
+
"OmniRAG",
|
|
26
|
+
]
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
# Chunking
|
|
2
|
+
|
|
3
|
+
The chunker consumes OmniLoad's canonical `Document` output. It does not replace or change the ingestion connectors.
|
|
4
|
+
|
|
5
|
+
## Automatic mode
|
|
6
|
+
|
|
7
|
+
```python
|
|
8
|
+
from core.loader import Loader
|
|
9
|
+
|
|
10
|
+
loader = Loader(connectors_dir="connectors")
|
|
11
|
+
chunks = loader.process_files(["report.pdf"], auto=True)
|
|
12
|
+
|
|
13
|
+
result = loader.process_files(
|
|
14
|
+
["report.pdf"],
|
|
15
|
+
auto=True,
|
|
16
|
+
chunking_method="Recursive Character Chunking",
|
|
17
|
+
chunking_options={"chunk_size": 1200, "overlap": 120},
|
|
18
|
+
include_documents=True,
|
|
19
|
+
)
|
|
20
|
+
documents = result["documents"]
|
|
21
|
+
chunks = result["chunks"]
|
|
22
|
+
```
|
|
23
|
+
|
|
24
|
+
`loader.load(path, auto=True)` is the single-file equivalent. For registered sources, `loader.process_connector(name, config, auto=True)` ingests and chunks the connector's output. SQLite accepts `{"file_path": "data.sqlite"}`; legacy database connectors accept connection settings such as `{"host": ..., "database": ..., "user": ..., "password": ...}`. Connector-specific drivers must be installed in the active Python environment for this direct API.
|
|
25
|
+
|
|
26
|
+
Without `auto=True`, `load()` and `process_files()` retain their document-returning behavior, and `process_connector()` returns normalized documents.
|
|
27
|
+
|
|
28
|
+
## Manual mode
|
|
29
|
+
|
|
30
|
+
```python
|
|
31
|
+
from core.loader import Loader
|
|
32
|
+
from chunking import Chunker
|
|
33
|
+
|
|
34
|
+
loader = Loader(connectors_dir="connectors")
|
|
35
|
+
documents = loader.process_files(["report.pdf"])
|
|
36
|
+
chunker = Chunker("Sentence Chunking", chunk_size=800)
|
|
37
|
+
chunks = chunker.chunk_documents(documents)
|
|
38
|
+
```
|
|
39
|
+
|
|
40
|
+
`Chunker.available_methods()` lists all registered strategy names. Each result is a `Chunk` with `id`, `document_id`, `content`, and provenance metadata, including the source document metadata and selected strategy.
|
|
41
|
+
|
|
42
|
+
## Strategies and capabilities
|
|
43
|
+
|
|
44
|
+
Every named strategy has its own module under `chunking/methods/`, for example `fixed_character_chunking.py` and `json_object_chunking.py`. Each module exports a `Strategy` class. `chunking.methods` discovers these modules and registers their strategy names; `Chunker` dispatches each registered strategy to the shared parser and boundary engine. This keeps methods separately addressable without duplicating common parsing code.
|
|
45
|
+
|
|
46
|
+
The registry includes the requested general, semantic, hierarchical, structure-aware, Markdown, HTML/XML, JSON/YAML, document, table/database, code, conversation, audio/video/image, graph, and adaptive strategy names. Format-aware methods use structure available in document text and metadata. They cannot recover layout or binary-media details that an ingestion connector did not retain.
|
|
47
|
+
|
|
48
|
+
To add a strategy, create `chunking/methods/<method_name>.py`, subclass `ChunkingStrategy` as `Strategy`, and set its `name` and a supported `group`. The package discovers the file automatically.
|
|
49
|
+
|
|
50
|
+
Embedding-based boundaries accept `embedder(sentences)`, which must return one numeric vector per sentence. Similarity methods accept `similarity_fn(left, right)`. LLM semantic chunking accepts `llm_segmenter(document_text)`, returning chunk strings. Without these callbacks, semantic methods use deterministic sentence/size boundaries and mark chunks with `semantic_fallback=True`; no model is loaded implicitly.
|
|
@@ -0,0 +1,17 @@
|
|
|
1
|
+
"""Independent chunking framework (no dependency on the ingestion layer).
|
|
2
|
+
|
|
3
|
+
from ragbench.chunking import chunk
|
|
4
|
+
chunks = chunk("some text", method="auto")
|
|
5
|
+
"""
|
|
6
|
+
from typing import Any, List
|
|
7
|
+
|
|
8
|
+
from ragbench.chunking.chunker import CHUNKING_METHODS, REGISTRY, Chunker
|
|
9
|
+
from ragbench.chunking.models import Chunk
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def chunk(data: Any, method: str = "auto", **options: Any) -> List[Chunk]:
|
|
13
|
+
"""Chunk a string, dict, Document-like object, or list of them."""
|
|
14
|
+
return Chunker(method, **options).chunk(data)
|
|
15
|
+
|
|
16
|
+
|
|
17
|
+
__all__ = ["CHUNKING_METHODS", "REGISTRY", "Chunk", "Chunker", "chunk"]
|
|
@@ -0,0 +1,507 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import ast
|
|
4
|
+
import csv
|
|
5
|
+
import hashlib
|
|
6
|
+
import html.parser
|
|
7
|
+
import importlib
|
|
8
|
+
import io
|
|
9
|
+
import json
|
|
10
|
+
import re
|
|
11
|
+
import uuid
|
|
12
|
+
import xml.etree.ElementTree as ET
|
|
13
|
+
from typing import Any, Callable, Dict, Iterable, List, Mapping, Optional, Sequence, Tuple, cast
|
|
14
|
+
|
|
15
|
+
from ragbench.chunking.models import Chunk
|
|
16
|
+
from ragbench.chunking.registry import ChunkingRegistry
|
|
17
|
+
|
|
18
|
+
|
|
19
|
+
REGISTRY = ChunkingRegistry()
|
|
20
|
+
METHOD_REGISTRY = {name: REGISTRY.get(name) for name in REGISTRY.list_names()}
|
|
21
|
+
CHUNKING_METHODS: Dict[str, str] = REGISTRY.groups()
|
|
22
|
+
|
|
23
|
+
|
|
24
|
+
def _get(document: Any, key: str, default: Any = None) -> Any:
|
|
25
|
+
if isinstance(document, Mapping):
|
|
26
|
+
return cast(Mapping[str, Any], document).get(key, default)
|
|
27
|
+
return getattr(document, key, default)
|
|
28
|
+
|
|
29
|
+
|
|
30
|
+
def _plain_text(value: Any) -> str:
|
|
31
|
+
if value is None:
|
|
32
|
+
return ""
|
|
33
|
+
if isinstance(value, str):
|
|
34
|
+
return value
|
|
35
|
+
if isinstance(value, (dict, list, tuple)):
|
|
36
|
+
return json.dumps(value, ensure_ascii=False, default=str, indent=2)
|
|
37
|
+
return str(value)
|
|
38
|
+
|
|
39
|
+
|
|
40
|
+
def _sentences(text: str) -> List[str]:
|
|
41
|
+
return [part.strip() for part in re.split(r"(?<=[.!?])\s+|\n+", text) if part.strip()]
|
|
42
|
+
|
|
43
|
+
|
|
44
|
+
def _paragraphs(text: str) -> List[str]:
|
|
45
|
+
return [part.strip() for part in re.split(r"\n\s*\n", text) if part.strip()]
|
|
46
|
+
|
|
47
|
+
|
|
48
|
+
def _markdown_sections(text: str) -> List[Tuple[str, str]]:
|
|
49
|
+
sections: List[Tuple[str, str]] = []
|
|
50
|
+
title = "Document"
|
|
51
|
+
body: List[str] = []
|
|
52
|
+
for line in text.splitlines():
|
|
53
|
+
if re.match(r"^\s{0,3}#{1,6}\s+", line):
|
|
54
|
+
if body:
|
|
55
|
+
sections.append((title, "\n".join(body).strip()))
|
|
56
|
+
title = re.sub(r"^\s{0,3}#{1,6}\s+", "", line).strip()
|
|
57
|
+
body = [line]
|
|
58
|
+
else:
|
|
59
|
+
body.append(line)
|
|
60
|
+
if body:
|
|
61
|
+
sections.append((title, "\n".join(body).strip()))
|
|
62
|
+
return [(title, body) for title, body in sections if body]
|
|
63
|
+
|
|
64
|
+
|
|
65
|
+
class _HTMLTextExtractor(html.parser.HTMLParser):
|
|
66
|
+
def __init__(self) -> None:
|
|
67
|
+
super().__init__(convert_charrefs=True)
|
|
68
|
+
self.parts: List[str] = []
|
|
69
|
+
self.stack: List[str] = []
|
|
70
|
+
|
|
71
|
+
def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
|
|
72
|
+
_ = attrs
|
|
73
|
+
if tag in {"p", "div", "section", "article", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6", "br"}:
|
|
74
|
+
self.parts.append("\n")
|
|
75
|
+
self.stack.append(tag)
|
|
76
|
+
|
|
77
|
+
def handle_endtag(self, tag: str) -> None:
|
|
78
|
+
if tag in {"p", "div", "section", "article", "li", "tr", "h1", "h2", "h3", "h4", "h5", "h6"}:
|
|
79
|
+
self.parts.append("\n")
|
|
80
|
+
if tag in self.stack:
|
|
81
|
+
self.stack.reverse()
|
|
82
|
+
self.stack.remove(tag)
|
|
83
|
+
self.stack.reverse()
|
|
84
|
+
|
|
85
|
+
def handle_data(self, data: str) -> None:
|
|
86
|
+
if data.strip():
|
|
87
|
+
self.parts.append(data.strip())
|
|
88
|
+
|
|
89
|
+
|
|
90
|
+
class Chunker:
|
|
91
|
+
"""Turn OmniLoad Documents into chunks while retaining source provenance.
|
|
92
|
+
|
|
93
|
+
Semantic/model-dependent methods accept optional ``similarity_fn``,
|
|
94
|
+
``embedder``, or ``llm_segmenter`` callbacks. Without them, these methods
|
|
95
|
+
use deterministic text boundaries and mark the fallback in chunk metadata.
|
|
96
|
+
"""
|
|
97
|
+
|
|
98
|
+
def __init__(
|
|
99
|
+
self,
|
|
100
|
+
method: str = "Automatic Chunker Selection",
|
|
101
|
+
chunk_size: int = 1000,
|
|
102
|
+
overlap: int = 100,
|
|
103
|
+
token_counter: Optional[Callable[[str], int]] = None,
|
|
104
|
+
similarity_fn: Optional[Callable[[str, str], float]] = None,
|
|
105
|
+
embedder: Optional[Callable[[Sequence[str]], Sequence[Any]]] = None,
|
|
106
|
+
llm_segmenter: Optional[Callable[[str], Sequence[str]]] = None,
|
|
107
|
+
query: Optional[str] = None,
|
|
108
|
+
**options: Any,
|
|
109
|
+
) -> None:
|
|
110
|
+
if method.strip().lower() == "auto":
|
|
111
|
+
method = "Automatic Chunker Selection"
|
|
112
|
+
canonical = REGISTRY.resolve(method)
|
|
113
|
+
if chunk_size < 1:
|
|
114
|
+
raise ValueError("chunk_size must be greater than zero")
|
|
115
|
+
if overlap < 0:
|
|
116
|
+
raise ValueError("overlap cannot be negative")
|
|
117
|
+
self.method = canonical
|
|
118
|
+
self.strategy = METHOD_REGISTRY[canonical]
|
|
119
|
+
self.group = CHUNKING_METHODS[canonical]
|
|
120
|
+
self.chunk_size = chunk_size
|
|
121
|
+
self.overlap = min(overlap, chunk_size - 1)
|
|
122
|
+
self.token_counter: Callable[[str], int] = token_counter or _count_tokens
|
|
123
|
+
self.similarity_fn = similarity_fn
|
|
124
|
+
self.embedder = embedder
|
|
125
|
+
self.llm_segmenter = llm_segmenter
|
|
126
|
+
self.query = query
|
|
127
|
+
self.options: Dict[str, Any] = options
|
|
128
|
+
|
|
129
|
+
@classmethod
|
|
130
|
+
def available_methods(cls) -> List[str]:
|
|
131
|
+
return list(CHUNKING_METHODS)
|
|
132
|
+
|
|
133
|
+
def chunk_documents(self, documents: Iterable[Any]) -> List[Chunk]:
|
|
134
|
+
chunks: List[Chunk] = []
|
|
135
|
+
for document in documents:
|
|
136
|
+
chunks.extend(self.chunk_document(document))
|
|
137
|
+
return chunks
|
|
138
|
+
|
|
139
|
+
def chunk(self, data: Any, metadata: Optional[Dict[str, Any]] = None) -> List[Chunk]:
|
|
140
|
+
"""Standalone entry point: accepts a str, dict, Document-like object, or a list of them."""
|
|
141
|
+
if isinstance(data, (list, tuple)):
|
|
142
|
+
return self.chunk_documents(cast(Sequence[Any], data))
|
|
143
|
+
if isinstance(data, str) and metadata:
|
|
144
|
+
data = {"content": data, "metadata": metadata}
|
|
145
|
+
return self.chunk_document(data)
|
|
146
|
+
|
|
147
|
+
def chunk_document(self, document: Any) -> List[Chunk]:
|
|
148
|
+
content = _plain_text(_get(document, "content", document))
|
|
149
|
+
document_id = str(_get(document, "id", "")) or str(uuid.uuid4())
|
|
150
|
+
metadata_value = _get(document, "metadata", {})
|
|
151
|
+
document_metadata = cast(Dict[str, Any], metadata_value) if isinstance(metadata_value, dict) else {}
|
|
152
|
+
source = str(_get(document, "source", "") or "")
|
|
153
|
+
method = self.method
|
|
154
|
+
group = self.group
|
|
155
|
+
|
|
156
|
+
if method == "Automatic Chunker Selection":
|
|
157
|
+
method, group = self._select_method(content, document_metadata)
|
|
158
|
+
if method == "Content-Type-Aware Chunking":
|
|
159
|
+
method, group = self._select_method(content, document_metadata)
|
|
160
|
+
if method == "Query-Aware Chunking" and self.query:
|
|
161
|
+
group = "semantic"
|
|
162
|
+
if method in {"Embedding-Based Semantic Chunking", "LLM-Based Semantic Chunking", "Agentic Chunking"}:
|
|
163
|
+
if method == "LLM-Based Semantic Chunking" and self.llm_segmenter:
|
|
164
|
+
proposed = self.llm_segmenter(content)
|
|
165
|
+
if proposed:
|
|
166
|
+
return self._make_chunks(proposed, document_id, document_metadata, source, method)
|
|
167
|
+
|
|
168
|
+
strategy = METHOD_REGISTRY[method]
|
|
169
|
+
segments, extra_metadata = strategy.split(self, content, document_metadata)
|
|
170
|
+
return self._make_chunks(segments, document_id, document_metadata, source, method, extra_metadata)
|
|
171
|
+
|
|
172
|
+
def _select_method(self, content: str, metadata: Dict[str, Any]) -> Tuple[str, str]:
|
|
173
|
+
content_type = str(metadata.get("content_type", metadata.get("format", ""))).lower()
|
|
174
|
+
if content_type in {"markdown", "md"} or re.search(r"^#{1,6}\s", content, re.M):
|
|
175
|
+
return "Markdown Heading Chunking", "markdown_heading"
|
|
176
|
+
if content_type in {"json", "application/json"} or content.lstrip().startswith(("{", "[")):
|
|
177
|
+
return "JSON Object Chunking", "json_object"
|
|
178
|
+
if content_type in {"html", "xml"} or re.search(r"<(html|body|article|section|root)\b", content, re.I):
|
|
179
|
+
return "DOM-Based Chunking", "html_dom"
|
|
180
|
+
if "table" in content_type or metadata.get("table"):
|
|
181
|
+
return "Record-Based Chunking", "row"
|
|
182
|
+
return "Recursive Character Chunking", "recursive"
|
|
183
|
+
|
|
184
|
+
def _split(self, text: str, group: str, metadata: Dict[str, Any]) -> Tuple[List[str], Dict[str, Any]]:
|
|
185
|
+
if not text.strip():
|
|
186
|
+
return [], {}
|
|
187
|
+
if group == "fixed_character":
|
|
188
|
+
return self._fixed(text, self.chunk_size, self.overlap), {}
|
|
189
|
+
if group == "fixed_token":
|
|
190
|
+
return self._token_chunks(text), {}
|
|
191
|
+
if group == "sentence":
|
|
192
|
+
return self._group_units(_sentences(text), lambda item: len(item), self.chunk_size), {}
|
|
193
|
+
if group == "paragraph":
|
|
194
|
+
return _paragraphs(text), {}
|
|
195
|
+
if group == "recursive":
|
|
196
|
+
return self._recursive(text), {}
|
|
197
|
+
if group in {"semantic", "audio_semantic"}:
|
|
198
|
+
segments, fallback = self._semantic(text)
|
|
199
|
+
return segments, {"semantic_fallback": fallback} if fallback else {}
|
|
200
|
+
if group == "hierarchical":
|
|
201
|
+
return self._hierarchical(text), {"hierarchy": ["document", "section", "paragraph"]}
|
|
202
|
+
if group in {"heading", "markdown_heading", "markdown_code", "markdown_table", "pdf_section", "docx_heading"}:
|
|
203
|
+
if group == "markdown_code":
|
|
204
|
+
return self._markdown_blocks(text, r"```[\s\S]*?```"), {}
|
|
205
|
+
if group == "markdown_table":
|
|
206
|
+
return self._markdown_blocks(text, r"(?:^\|.*\|\s*\n){2,}"), {}
|
|
207
|
+
return [body for _, body in _markdown_sections(text)], {}
|
|
208
|
+
if group == "list":
|
|
209
|
+
return self._list_blocks(text), {}
|
|
210
|
+
if group in {"block", "docx_paragraph", "page", "pdf_layout", "pptx_slide", "video_scene", "repository"}:
|
|
211
|
+
delimiter = "\f" if group == "page" else r"\n(?=\s*(?:```|#{1,6}\s|(?:def|class)\s))"
|
|
212
|
+
if group == "pptx_slide":
|
|
213
|
+
delimiter = r"\n(?=(?:Slide\s+\d+|={3,}\s*Slide))"
|
|
214
|
+
if group == "video_scene":
|
|
215
|
+
delimiter = r"\n(?=(?:Scene|Shot)\s+\d+)"
|
|
216
|
+
parts = re.split(delimiter, text) if group == "page" else re.split(delimiter, text)
|
|
217
|
+
return [part.strip() for part in parts if part.strip()], {}
|
|
218
|
+
if group in {"html_dom", "html_section"}:
|
|
219
|
+
parser = _HTMLTextExtractor()
|
|
220
|
+
parser.feed(text)
|
|
221
|
+
plain = " ".join(" ".join(parser.parts).split())
|
|
222
|
+
segments = _paragraphs(plain) or [plain]
|
|
223
|
+
return segments, {"structure": group}
|
|
224
|
+
if group == "xml_tree":
|
|
225
|
+
try:
|
|
226
|
+
root = ET.fromstring(text)
|
|
227
|
+
elements = [" ".join(part.strip() for part in element.itertext() if part.strip()) for element in root.iter()]
|
|
228
|
+
return [item for item in elements if item], {"structure": "xml"}
|
|
229
|
+
except ET.ParseError:
|
|
230
|
+
return self._recursive(text), {"parse_fallback": True}
|
|
231
|
+
if group in {"json_object", "json_array", "nested", "yaml", "schema"}:
|
|
232
|
+
return self._structured(text, group), {"structure": group}
|
|
233
|
+
if group in {"row", "column", "table", "pdf_table", "docx_table"}:
|
|
234
|
+
return self._table_chunks(text, group), {"structure": group}
|
|
235
|
+
if group == "sql":
|
|
236
|
+
return [part.strip() for part in re.split(r";\s*(?:\n|$)", text) if part.strip()], {}
|
|
237
|
+
if group in {"ast", "dependency"}:
|
|
238
|
+
return self._code_chunks(text, group), {"structure": "python_ast"}
|
|
239
|
+
if group in {"message", "thread", "speaker", "time", "transcript"}:
|
|
240
|
+
return self._communication_chunks(text, group), {"structure": group}
|
|
241
|
+
if group in {"image_ocr", "image_region"}:
|
|
242
|
+
regions: Any = metadata.get("regions") or metadata.get("ocr")
|
|
243
|
+
if isinstance(regions, list) and regions:
|
|
244
|
+
region_items = cast(List[Any], regions)
|
|
245
|
+
return [_plain_text(region) for region in region_items], {"structure": "image_regions"}
|
|
246
|
+
return self._recursive(text), {"structure_fallback": "text"}
|
|
247
|
+
if group.startswith("graph_"):
|
|
248
|
+
return self._graph_chunks(text, group), {"structure": group}
|
|
249
|
+
return self._recursive(text), {}
|
|
250
|
+
|
|
251
|
+
def _fixed(self, text: str, size: int, overlap: int) -> List[str]:
|
|
252
|
+
output: List[str] = []
|
|
253
|
+
start = 0
|
|
254
|
+
while start < len(text):
|
|
255
|
+
end = min(start + size, len(text))
|
|
256
|
+
if end < len(text):
|
|
257
|
+
boundary = text.rfind(" ", start, end)
|
|
258
|
+
if boundary > start:
|
|
259
|
+
end = boundary
|
|
260
|
+
output.append(text[start:end].strip())
|
|
261
|
+
if end >= len(text):
|
|
262
|
+
break
|
|
263
|
+
start = max(start + 1, end - overlap)
|
|
264
|
+
return [part for part in output if part]
|
|
265
|
+
|
|
266
|
+
def _token_chunks(self, text: str) -> List[str]:
|
|
267
|
+
words = text.split()
|
|
268
|
+
output: List[str] = []
|
|
269
|
+
start = 0
|
|
270
|
+
while start < len(words):
|
|
271
|
+
end = start
|
|
272
|
+
while end < len(words) and self.token_counter(" ".join(words[start:end + 1])) <= self.chunk_size:
|
|
273
|
+
end += 1
|
|
274
|
+
if end == start:
|
|
275
|
+
end += 1
|
|
276
|
+
output.append(" ".join(words[start:end]))
|
|
277
|
+
if end >= len(words):
|
|
278
|
+
break
|
|
279
|
+
start = max(start + 1, end - self.overlap)
|
|
280
|
+
return output
|
|
281
|
+
|
|
282
|
+
def _recursive(self, text: str) -> List[str]:
|
|
283
|
+
output: List[str] = []
|
|
284
|
+
for paragraph in _paragraphs(text):
|
|
285
|
+
if len(paragraph) <= self.chunk_size:
|
|
286
|
+
output.append(paragraph)
|
|
287
|
+
else:
|
|
288
|
+
units = _sentences(paragraph)
|
|
289
|
+
output.extend(self._group_units(units, len, self.chunk_size) if units else self._fixed(paragraph, self.chunk_size, self.overlap))
|
|
290
|
+
return output
|
|
291
|
+
|
|
292
|
+
@staticmethod
|
|
293
|
+
def _group_units(units: Sequence[str], measure: Callable[[str], int], size: int) -> List[str]:
|
|
294
|
+
output: List[str] = []
|
|
295
|
+
current: List[str] = []
|
|
296
|
+
length = 0
|
|
297
|
+
for unit in units:
|
|
298
|
+
unit_length = measure(unit)
|
|
299
|
+
if current and length + unit_length + 1 > size:
|
|
300
|
+
output.append(" ".join(current))
|
|
301
|
+
current = []
|
|
302
|
+
length = 0
|
|
303
|
+
current.append(unit)
|
|
304
|
+
length += unit_length + 1
|
|
305
|
+
if current:
|
|
306
|
+
output.append(" ".join(current))
|
|
307
|
+
return output
|
|
308
|
+
|
|
309
|
+
def _semantic(self, text: str) -> Tuple[List[str], bool]:
|
|
310
|
+
units = _sentences(text)
|
|
311
|
+
if not units:
|
|
312
|
+
return [], False
|
|
313
|
+
embeddings = list(self.embedder(units)) if self.embedder else []
|
|
314
|
+
output: List[str] = []
|
|
315
|
+
current = [units[0]]
|
|
316
|
+
for index, (previous, unit) in enumerate(zip(units, units[1:])):
|
|
317
|
+
if self.similarity_fn:
|
|
318
|
+
similarity = self.similarity_fn(previous, unit)
|
|
319
|
+
elif len(embeddings) == len(units):
|
|
320
|
+
left, right = embeddings[index], embeddings[index + 1]
|
|
321
|
+
dot = sum(float(a) * float(b) for a, b in zip(left, right))
|
|
322
|
+
left_norm = sum(float(value) ** 2 for value in left) ** 0.5
|
|
323
|
+
right_norm = sum(float(value) ** 2 for value in right) ** 0.5
|
|
324
|
+
similarity = dot / (left_norm * right_norm) if left_norm and right_norm else 0.0
|
|
325
|
+
else:
|
|
326
|
+
similarity = 1.0
|
|
327
|
+
similar = similarity >= self.options.get("similarity_threshold", 0.5)
|
|
328
|
+
if not similar or len(" ".join(current)) + len(unit) > self.chunk_size:
|
|
329
|
+
output.append(" ".join(current))
|
|
330
|
+
current = []
|
|
331
|
+
current.append(unit)
|
|
332
|
+
if current:
|
|
333
|
+
output.append(" ".join(current))
|
|
334
|
+
return output, self.similarity_fn is None and not embeddings and self.llm_segmenter is None
|
|
335
|
+
|
|
336
|
+
def _hierarchical(self, text: str) -> List[str]:
|
|
337
|
+
output: List[str] = []
|
|
338
|
+
for _, section in _markdown_sections(text):
|
|
339
|
+
output.extend(_paragraphs(section) or [section])
|
|
340
|
+
return output
|
|
341
|
+
|
|
342
|
+
def _markdown_blocks(self, text: str, pattern: str) -> List[str]:
|
|
343
|
+
matches = list(re.finditer(pattern, text, re.M))
|
|
344
|
+
if not matches:
|
|
345
|
+
return self._recursive(text)
|
|
346
|
+
parts: List[str] = []
|
|
347
|
+
cursor = 0
|
|
348
|
+
for match in matches:
|
|
349
|
+
if match.start() > cursor:
|
|
350
|
+
parts.extend(_paragraphs(text[cursor:match.start()]))
|
|
351
|
+
parts.append(match.group(0).strip())
|
|
352
|
+
cursor = match.end()
|
|
353
|
+
if cursor < len(text):
|
|
354
|
+
parts.extend(_paragraphs(text[cursor:]))
|
|
355
|
+
return [part for part in parts if part]
|
|
356
|
+
|
|
357
|
+
@staticmethod
|
|
358
|
+
def _list_blocks(text: str) -> List[str]:
|
|
359
|
+
blocks: List[str] = []
|
|
360
|
+
for line in text.splitlines():
|
|
361
|
+
if re.match(r"^\s*(?:[-*+] |\d+[.)] )", line) and blocks:
|
|
362
|
+
blocks[-1] += "\n" + line
|
|
363
|
+
elif re.match(r"^\s*(?:[-*+] |\d+[.)] )", line):
|
|
364
|
+
blocks.append(line)
|
|
365
|
+
elif line.strip():
|
|
366
|
+
blocks.append(line.strip())
|
|
367
|
+
return blocks
|
|
368
|
+
|
|
369
|
+
def _structured(self, text: str, group: str) -> List[str]:
|
|
370
|
+
try:
|
|
371
|
+
if group == "yaml":
|
|
372
|
+
yaml_module: Any = importlib.import_module("yaml")
|
|
373
|
+
value: Any = yaml_module.safe_load(text)
|
|
374
|
+
else:
|
|
375
|
+
value = json.loads(text)
|
|
376
|
+
except Exception: # yaml.YAMLError is not a ValueError; any parse failure -> text fallback
|
|
377
|
+
return self._recursive(text)
|
|
378
|
+
if group == "yaml" and not isinstance(value, (dict, list)):
|
|
379
|
+
return self._recursive(text) # plain prose that merely parses as a YAML scalar
|
|
380
|
+
if isinstance(value, list):
|
|
381
|
+
items = cast(List[Any], value)
|
|
382
|
+
return [json.dumps(item, ensure_ascii=False, default=str) for item in items] or ["[]"]
|
|
383
|
+
if isinstance(value, dict):
|
|
384
|
+
mapping = cast(Dict[str, Any], value)
|
|
385
|
+
if group in {"json_object", "nested", "schema"}:
|
|
386
|
+
return [json.dumps({key: item}, ensure_ascii=False, default=str) for key, item in mapping.items()] or ["{}"]
|
|
387
|
+
return [json.dumps(value, ensure_ascii=False, default=str)]
|
|
388
|
+
return [json.dumps(value, ensure_ascii=False, default=str)]
|
|
389
|
+
|
|
390
|
+
def _table_chunks(self, text: str, group: str) -> List[str]:
|
|
391
|
+
lines = [line for line in text.splitlines() if line.strip()]
|
|
392
|
+
if not lines:
|
|
393
|
+
return []
|
|
394
|
+
try:
|
|
395
|
+
rows: List[List[str]] = list(csv.reader(io.StringIO("\n".join(lines))))
|
|
396
|
+
except csv.Error:
|
|
397
|
+
rows = [line.split("\t") for line in lines]
|
|
398
|
+
if group == "column":
|
|
399
|
+
return ["\n".join(row[index] for row in rows if index < len(row)) for index in range(max(map(len, rows)))]
|
|
400
|
+
if group in {"table", "schema", "pdf_table", "docx_table"}:
|
|
401
|
+
return ["\n".join(lines)]
|
|
402
|
+
width = max(1, int(self.options.get("rows_per_chunk", 10)))
|
|
403
|
+
header = rows[0] if group == "table" else []
|
|
404
|
+
data_rows = rows[1:] if header else rows
|
|
405
|
+
output: List[str] = []
|
|
406
|
+
for index in range(0, len(data_rows), width):
|
|
407
|
+
group_rows = ([header] if header else []) + data_rows[index:index + width]
|
|
408
|
+
output.append("\n".join(",".join(row) for row in group_rows))
|
|
409
|
+
return output
|
|
410
|
+
|
|
411
|
+
def _code_chunks(self, text: str, group: str) -> List[str]:
|
|
412
|
+
try:
|
|
413
|
+
tree = ast.parse(text)
|
|
414
|
+
except SyntaxError:
|
|
415
|
+
return self._recursive(text)
|
|
416
|
+
nodes = []
|
|
417
|
+
if group == "dependency":
|
|
418
|
+
nodes = [node for node in ast.walk(tree) if isinstance(node, (ast.Import, ast.ImportFrom))]
|
|
419
|
+
if not nodes:
|
|
420
|
+
return [text]
|
|
421
|
+
else:
|
|
422
|
+
nodes = [node for node in tree.body if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef))]
|
|
423
|
+
if not nodes:
|
|
424
|
+
return [text]
|
|
425
|
+
return [ast.get_source_segment(text, node) or "" for node in nodes if ast.get_source_segment(text, node)]
|
|
426
|
+
|
|
427
|
+
def _communication_chunks(self, text: str, group: str) -> List[str]:
|
|
428
|
+
messages = [line.strip() for line in text.splitlines() if line.strip()]
|
|
429
|
+
if group == "speaker":
|
|
430
|
+
speakers: Dict[str, List[str]] = {}
|
|
431
|
+
for message in messages:
|
|
432
|
+
match = re.match(r"\[?([^:\]]+)\]?:\s*(.*)", message)
|
|
433
|
+
key, body = (match.group(1), match.group(2)) if match else ("unknown", message)
|
|
434
|
+
speakers.setdefault(key, []).append(body)
|
|
435
|
+
return [f"{speaker}: " + "\n".join(items) for speaker, items in speakers.items()]
|
|
436
|
+
if group == "time":
|
|
437
|
+
window = int(self.options.get("window_seconds", 60))
|
|
438
|
+
chunks: Dict[int, List[str]] = {}
|
|
439
|
+
for message in messages:
|
|
440
|
+
match = re.search(r"(?:\[)?(\d{1,2}):(\d{2})(?::(\d{2}))?(?:\])?", message)
|
|
441
|
+
seconds = int(match.group(1)) * 60 + int(match.group(2)) if match else 0
|
|
442
|
+
chunks.setdefault(seconds // max(1, window), []).append(message)
|
|
443
|
+
return ["\n".join(items) for items in chunks.values()]
|
|
444
|
+
if group == "thread":
|
|
445
|
+
return [part.strip() for part in re.split(r"\n(?=Thread\s|Subject:)", text) if part.strip()]
|
|
446
|
+
return ["\n".join(messages[index:index + 10]) for index in range(0, len(messages), 10)]
|
|
447
|
+
|
|
448
|
+
def _graph_chunks(self, text: str, group: str) -> List[str]:
|
|
449
|
+
try:
|
|
450
|
+
parsed: Any = json.loads(text)
|
|
451
|
+
except ValueError:
|
|
452
|
+
return self._recursive(text)
|
|
453
|
+
if not isinstance(parsed, dict):
|
|
454
|
+
return [text]
|
|
455
|
+
graph = cast(Dict[str, Any], parsed)
|
|
456
|
+
nodes_value: Any = graph.get("nodes", [])
|
|
457
|
+
edges_value: Any = graph.get("edges", [])
|
|
458
|
+
nodes = cast(List[Any], nodes_value) if isinstance(nodes_value, list) else []
|
|
459
|
+
edges = cast(List[Any], edges_value) if isinstance(edges_value, list) else []
|
|
460
|
+
values: List[Any]
|
|
461
|
+
if group == "graph_node":
|
|
462
|
+
values = nodes
|
|
463
|
+
elif group == "graph_edge":
|
|
464
|
+
values = edges
|
|
465
|
+
else:
|
|
466
|
+
values = nodes or edges
|
|
467
|
+
return [json.dumps(value, ensure_ascii=False, default=str) for value in values] or [text]
|
|
468
|
+
|
|
469
|
+
def _make_chunks(
|
|
470
|
+
self,
|
|
471
|
+
segments: Iterable[Any],
|
|
472
|
+
document_id: str,
|
|
473
|
+
document_metadata: Dict[str, Any],
|
|
474
|
+
source: str,
|
|
475
|
+
method: str,
|
|
476
|
+
extra_metadata: Optional[Dict[str, Any]] = None,
|
|
477
|
+
) -> List[Chunk]:
|
|
478
|
+
chunks: List[Chunk] = []
|
|
479
|
+
parent_id = document_metadata.get("parent_id", document_id)
|
|
480
|
+
for index, segment in enumerate(segments):
|
|
481
|
+
content = _plain_text(segment).strip()
|
|
482
|
+
if not content:
|
|
483
|
+
continue
|
|
484
|
+
metadata: Dict[str, Any] = {
|
|
485
|
+
"chunk_index": len(chunks),
|
|
486
|
+
"source": source,
|
|
487
|
+
"strategy": method,
|
|
488
|
+
"document_metadata": document_metadata,
|
|
489
|
+
"parent_id": parent_id,
|
|
490
|
+
}
|
|
491
|
+
if extra_metadata:
|
|
492
|
+
metadata.update(extra_metadata)
|
|
493
|
+
metadata["chunk_index"] = len(chunks)
|
|
494
|
+
chunks.append(Chunk(
|
|
495
|
+
id=hashlib.sha256(f"{document_id}:{method}:{index}:{content}".encode("utf-8")).hexdigest(),
|
|
496
|
+
document_id=document_id,
|
|
497
|
+
content=content,
|
|
498
|
+
metadata=metadata,
|
|
499
|
+
))
|
|
500
|
+
count = len(chunks)
|
|
501
|
+
for chunk in chunks:
|
|
502
|
+
chunk.metadata["chunk_count"] = count
|
|
503
|
+
return chunks
|
|
504
|
+
|
|
505
|
+
|
|
506
|
+
def _count_tokens(text: str) -> int:
|
|
507
|
+
return len(text.split())
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
"""Chunking methods: one folder per method (see chunking/registry.py)."""
|
|
2
|
+
from typing import Dict
|
|
3
|
+
|
|
4
|
+
from ragbench.chunking.methods.base import ChunkingStrategy
|
|
5
|
+
|
|
6
|
+
__all__ = ["ChunkingStrategy", "METHOD_REGISTRY"]
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
def __getattr__(attr: str):
|
|
10
|
+
# Lazy so that importing a single method folder never triggers a full scan.
|
|
11
|
+
if attr == "METHOD_REGISTRY":
|
|
12
|
+
from ragbench.chunking.registry import ChunkingRegistry
|
|
13
|
+
|
|
14
|
+
registry = ChunkingRegistry()
|
|
15
|
+
return {n: registry.get(n) for n in registry.list_names()}
|
|
16
|
+
raise AttributeError(attr)
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "Adaptive Chunking",
|
|
3
|
+
"slug": "adaptive_chunking",
|
|
4
|
+
"group": "semantic",
|
|
5
|
+
"category": "semantic",
|
|
6
|
+
"extensions": [],
|
|
7
|
+
"description": "Adaptive Chunking (semantic / semantic implementation)",
|
|
8
|
+
"entry_point": "strategy.Strategy",
|
|
9
|
+
"requires_callbacks": []
|
|
10
|
+
}
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "Agentic Chunking",
|
|
3
|
+
"slug": "agentic_chunking",
|
|
4
|
+
"group": "semantic",
|
|
5
|
+
"category": "semantic",
|
|
6
|
+
"extensions": [],
|
|
7
|
+
"description": "Agentic Chunking (semantic / semantic implementation)",
|
|
8
|
+
"entry_point": "strategy.Strategy",
|
|
9
|
+
"requires_callbacks": []
|
|
10
|
+
}
|