haystack-ai 3.3.0rc1__tar.gz → 3.4.0.dev20261002000327__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/PKG-INFO +1 -1
- haystack_ai-3.4.0.dev20261002000327/VERSION.txt +1 -0
- haystack_ai-3.4.0.dev20261002000327/haystack/components/preprocessors/_page_numbers.py +38 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/document_splitter.py +4 -2
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/embedding_based_document_splitter.py +8 -6
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/markdown_header_splitter.py +19 -6
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/recursive_splitter.py +101 -48
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/sentence_tokenizer.py +1 -1
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/lost_in_the_middle.py +4 -4
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/telemetry/_telemetry.py +1 -1
- haystack_ai-3.3.0rc1/VERSION.txt +0 -1
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/.gitignore +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/LICENSE +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/README.md +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/agent.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/state/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/state/state.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/state/state_utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/tool_calling.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/builders/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/builders/answer_builder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/builders/chat_prompt_builder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/builders/prompt_builder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/caching/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/caching/cache_checker.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/csv.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/docx.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/file_to_file_content.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/html.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/document_to_image.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/file_to_document.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/file_to_image.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/image_utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/pdf_to_image.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/json.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/markdown.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/msg.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/multi_file_converter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/output_adapter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/pdfminer.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/pptx.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/pypdf.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/txt.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/xlsx.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/azure_document_embedder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/azure_text_embedder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/mock_document_embedder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/mock_text_embedder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/mock_utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/openai_document_embedder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/openai_text_embedder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/types/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/types/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/answer_exact_match.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/context_relevance.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/document_map.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/document_mrr.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/document_ndcg.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/document_recall.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/faithfulness.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/llm_evaluator.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/sas_evaluator.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/image/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/image/llm_document_content_extractor.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/llm_metadata_extractor.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/regex_text_extractor.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/fetchers/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/fetchers/link_content.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/azure.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/azure_responses.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/fallback.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/llm.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/mock.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/openai.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/openai_responses.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/types/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/types/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/openai_image_generator.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/answer_joiner.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/branch.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/document_joiner.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/list_joiner.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/string_joiner.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/csv_document_cleaner.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/csv_document_splitter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/document_cleaner.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/document_preprocessor.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/hierarchical_document_splitter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/python_code_splitter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/text_cleaner.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/query/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/query/query_expander.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/llm_ranker.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/meta_field.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/meta_field_grouping_ranker.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/auto_merging_retriever.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/filter_retriever.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/in_memory/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/in_memory/bm25_retriever.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/in_memory/embedding_retriever.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/multi_query_embedding_retriever.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/multi_query_text_retriever.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/multi_retriever.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/sentence_window_retriever.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/text_embedding_retriever.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/types/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/types/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/conditional_router.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/document_length_router.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/document_type_router.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/file_type_router.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/llm_messages_router.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/metadata_router.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/samplers/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/samplers/top_p.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/validators/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/validators/json_schema.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/writers/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/writers/document_writer.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/component/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/component/component.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/component/sockets.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/component/types.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/errors.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/base.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/breakpoint.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/component_checks.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/descriptions.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/draw.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/pipeline.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/serialization.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/serialization_security.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/super_component/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/super_component/super_component.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/super_component/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/type_utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/data/abbreviations/de.txt +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/data/abbreviations/en.txt +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/answer.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/breakpoints.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/byte_stream.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/chat_message.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/document.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/file_content.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/image_content.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/skill_info.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/sparse_embedding.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/streaming_chunk.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/errors/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/errors/errors.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/in_memory/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/in_memory/document_store.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/types/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/types/filter_policy.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/types/policy.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/types/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/errors.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/evaluation/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/evaluation/eval_run_result.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/budget/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/budget/hooks.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/AGENTS.md +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/CLAUDE.md +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/hooks.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/sliding_window.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/summarization.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/tool_result_pruning.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/types/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/types/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/from_function.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/dataclasses.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/hooks.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/policies.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/strategies.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/types/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/types/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/user_interfaces.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/invocation.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/hooks.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/policies.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/stores.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/types/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/types/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/lazy_imports.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/logging.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/marshal/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/marshal/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/marshal/yaml.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/py.typed +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/file_system/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/file_system/skill_store.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/types/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/types/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/telemetry/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/telemetry/_environment.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/callable_serialization/random_callable.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/document_store.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/document_store_async.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/factory.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/accumulate.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/add_value.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/concatenate.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/double.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/fstring.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/future_annotations.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/greet.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/hello.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/joiner.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/parity.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/remainder.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/repeat.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/subtract.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/sum.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/text_splitter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/threshold.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/telemetry.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/test_utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/approximate_counter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/openai_counter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/tiktoken_counter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/types/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/types/protocol.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/agent_tool.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/component_tool.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/errors.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/from_function.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/parameters_schema_utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/pipeline_tool.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/searchable_toolset.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/serde_utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/skills/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/skills/skill_toolset.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/tool.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/tool_types.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/toolset.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tracing/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tracing/logging_tracer.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tracing/tracer.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tracing/utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/__init__.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/async_utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/auth.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/azure.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/base_serialization.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/callable_serialization.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/dataclasses.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/deserialization.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/device.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/experimental.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/filters.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/hf.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/http_client.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/jinja2_chat_extension.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/jinja2_extensions.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/jinja2_sandbox.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/jupyter.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/misc.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/requests_utils.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/type_serialization.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/url_validation.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/version.py +0 -0
- {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/pyproject.toml +0 -0
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: haystack-ai
|
|
3
|
-
Version: 3.
|
|
3
|
+
Version: 3.4.0.dev20261002000327
|
|
4
4
|
Summary: LLM framework to build customizable, production-ready LLM applications. Connect components (models, vector DBs, file converters) to pipelines or agents that can interact with your data.
|
|
5
5
|
Project-URL: CI: GitHub, https://github.com/deepset-ai/haystack/actions
|
|
6
6
|
Project-URL: Docs: RTD, https://haystack.deepset.ai/overview/intro
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
3.4.0.dev20261002000327
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
|
|
2
|
+
#
|
|
3
|
+
# SPDX-License-Identifier: Apache-2.0
|
|
4
|
+
|
|
5
|
+
"""
|
|
6
|
+
The `page_number` convention shared by the splitters.
|
|
7
|
+
|
|
8
|
+
Splitters record the page a chunk came from in its `page_number` metadata field, counting page breaks
|
|
9
|
+
(form feed, "\f") in the source document. The page reported is the one the chunk's first non-page-break
|
|
10
|
+
character lies on: a chunk that spans a page break belongs to the page it starts on, and page breaks a
|
|
11
|
+
chunk opens with belong to that chunk's own page rather than to the page it was carved out of.
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
|
|
15
|
+
def _leading_page_breaks(text: str, page_break_character: str = "\f") -> int:
|
|
16
|
+
"""
|
|
17
|
+
Count how many pages a chunk's text starts past the page its first character is on.
|
|
18
|
+
|
|
19
|
+
This is the run of page breaks the chunk opens with, except for a chunk made up entirely of page
|
|
20
|
+
breaks: that chunk is an empty page and stays on the page it starts on, rather than being pushed onto
|
|
21
|
+
the following one. `split_by="page"` emits such chunks for every blank page.
|
|
22
|
+
|
|
23
|
+
:param text: The chunk to inspect.
|
|
24
|
+
:param page_break_character: The character sequence marking a page break.
|
|
25
|
+
:returns: The number of pages to advance, 0 if the chunk starts with anything but a page break or
|
|
26
|
+
consists only of page breaks.
|
|
27
|
+
"""
|
|
28
|
+
if not page_break_character:
|
|
29
|
+
return 0
|
|
30
|
+
|
|
31
|
+
offset = 0
|
|
32
|
+
while text.startswith(page_break_character, offset):
|
|
33
|
+
offset += len(page_break_character)
|
|
34
|
+
|
|
35
|
+
# Nothing but page breaks: an empty page, which belongs to the page it starts on.
|
|
36
|
+
if offset >= len(text):
|
|
37
|
+
return 0
|
|
38
|
+
return offset // len(page_break_character)
|
|
@@ -9,6 +9,7 @@ from typing import Any, Literal
|
|
|
9
9
|
from more_itertools import windowed
|
|
10
10
|
|
|
11
11
|
from haystack import Document, component, logging
|
|
12
|
+
from haystack.components.preprocessors._page_numbers import _leading_page_breaks
|
|
12
13
|
from haystack.components.preprocessors.sentence_tokenizer import Language, SentenceSplitter, nltk_imports
|
|
13
14
|
from haystack.core.serialization import default_from_dict, default_to_dict
|
|
14
15
|
from haystack.lazy_imports import LazyImport
|
|
@@ -199,7 +200,8 @@ class DocumentSplitter:
|
|
|
199
200
|
:returns: A dictionary with the following key:
|
|
200
201
|
- `documents`: List of documents with the split texts. Each document includes:
|
|
201
202
|
- A metadata field `source_id` to track the original document.
|
|
202
|
-
- A metadata field `page_number`
|
|
203
|
+
- A metadata field `page_number` with the page the chunk starts on, counting form feed
|
|
204
|
+
("\f") characters in the original document.
|
|
203
205
|
- All other metadata copied from the original document.
|
|
204
206
|
|
|
205
207
|
:raises TypeError: if the input is not a list of Documents.
|
|
@@ -450,7 +452,7 @@ class DocumentSplitter:
|
|
|
450
452
|
|
|
451
453
|
for i, (txt, split_idx) in enumerate(zip(text_splits, splits_start_idxs, strict=True)):
|
|
452
454
|
copied_meta = deepcopy(meta)
|
|
453
|
-
copied_meta["page_number"] = splits_pages[i]
|
|
455
|
+
copied_meta["page_number"] = splits_pages[i] + _leading_page_breaks(txt)
|
|
454
456
|
copied_meta["split_id"] = i
|
|
455
457
|
copied_meta["split_idx_start"] = split_idx
|
|
456
458
|
doc = Document(content=txt, meta=copied_meta)
|
|
@@ -12,6 +12,7 @@ import numpy as np
|
|
|
12
12
|
|
|
13
13
|
from haystack import Document, component, logging
|
|
14
14
|
from haystack.components.embedders.types import DocumentEmbedder
|
|
15
|
+
from haystack.components.preprocessors._page_numbers import _leading_page_breaks
|
|
15
16
|
from haystack.components.preprocessors.sentence_tokenizer import Language, SentenceSplitter
|
|
16
17
|
from haystack.core.serialization import component_to_dict, default_from_dict, default_to_dict
|
|
17
18
|
from haystack.utils.async_utils import _execute_component_async
|
|
@@ -179,7 +180,8 @@ class EmbeddingBasedDocumentSplitter:
|
|
|
179
180
|
- A metadata field `source_id` to track the original document.
|
|
180
181
|
- A metadata field `split_id` to track the split number.
|
|
181
182
|
- A metadata field `split_idx_start` with the character offset of the chunk in the original document.
|
|
182
|
-
- A metadata field `page_number`
|
|
183
|
+
- A metadata field `page_number` with the page the chunk starts on, counting form feed
|
|
184
|
+
("\f") characters in the original document.
|
|
183
185
|
- All other metadata copied from the original document.
|
|
184
186
|
|
|
185
187
|
:raises RuntimeError: If the component wasn't warmed up.
|
|
@@ -220,7 +222,8 @@ class EmbeddingBasedDocumentSplitter:
|
|
|
220
222
|
- A metadata field `source_id` to track the original document.
|
|
221
223
|
- A metadata field `split_id` to track the split number.
|
|
222
224
|
- A metadata field `split_idx_start` with the character offset of the chunk in the original document.
|
|
223
|
-
- A metadata field `page_number`
|
|
225
|
+
- A metadata field `page_number` with the page the chunk starts on, counting form feed
|
|
226
|
+
("\f") characters in the original document.
|
|
224
227
|
- All other metadata copied from the original document.
|
|
225
228
|
|
|
226
229
|
:raises RuntimeError: If the component wasn't warmed up.
|
|
@@ -549,12 +552,11 @@ class EmbeddingBasedDocumentSplitter:
|
|
|
549
552
|
split_meta["split_id"] = i
|
|
550
553
|
split_meta["split_idx_start"] = current_char_pos
|
|
551
554
|
|
|
552
|
-
# Calculate page number for this split
|
|
553
|
-
# Count page breaks in the split itself
|
|
554
555
|
page_breaks_in_split = split_text.count("\f")
|
|
555
556
|
|
|
556
|
-
#
|
|
557
|
-
|
|
557
|
+
# current_page is the page this split's first character is on; a split that opens with page
|
|
558
|
+
# breaks starts its text on a later page.
|
|
559
|
+
split_meta["page_number"] = current_page + _leading_page_breaks(split_text)
|
|
558
560
|
|
|
559
561
|
doc = Document(content=split_text, meta=split_meta)
|
|
560
562
|
documents.append(doc)
|
|
@@ -8,6 +8,7 @@ from typing import Literal
|
|
|
8
8
|
|
|
9
9
|
from haystack import Document, component, logging
|
|
10
10
|
from haystack.components.preprocessors import DocumentSplitter
|
|
11
|
+
from haystack.components.preprocessors._page_numbers import _leading_page_breaks
|
|
11
12
|
|
|
12
13
|
logger = logging.getLogger(__name__)
|
|
13
14
|
|
|
@@ -294,8 +295,12 @@ class MarkdownHeaderSplitter:
|
|
|
294
295
|
secondary_content_start_idx = header_match.end()
|
|
295
296
|
content_for_splitting = doc.content[secondary_content_start_idx:]
|
|
296
297
|
|
|
297
|
-
# The page this header chunk
|
|
298
|
-
|
|
298
|
+
# The page this header chunk's first character is on; its splits are numbered relative to it.
|
|
299
|
+
# doc.meta holds the page the chunk's *text* starts on, which already counts the breaks the chunk
|
|
300
|
+
# opens with, and page_break_ends below counts those same breaks again - so take them back off.
|
|
301
|
+
chunk_start_page = doc.meta.get("page_number", 1) - _leading_page_breaks(
|
|
302
|
+
doc.content, self.page_break_character
|
|
303
|
+
)
|
|
299
304
|
|
|
300
305
|
clean_meta = {k: v for k, v in doc.meta.items() if k != "split_id"}
|
|
301
306
|
|
|
@@ -312,7 +317,11 @@ class MarkdownHeaderSplitter:
|
|
|
312
317
|
while page_break_count < len(page_break_ends) and page_break_ends[page_break_count] <= split_start_idx:
|
|
313
318
|
page_break_count += 1
|
|
314
319
|
|
|
315
|
-
split.meta["page_number"] =
|
|
320
|
+
split.meta["page_number"] = (
|
|
321
|
+
chunk_start_page
|
|
322
|
+
+ page_break_count
|
|
323
|
+
+ _leading_page_breaks(split.content or "", self.page_break_character)
|
|
324
|
+
)
|
|
316
325
|
split.meta["split_id"] = current_split_id
|
|
317
326
|
if "source_id" in doc.meta:
|
|
318
327
|
split.meta["source_id"] = doc.meta["source_id"]
|
|
@@ -358,8 +367,11 @@ class MarkdownHeaderSplitter:
|
|
|
358
367
|
)
|
|
359
368
|
for split_idx, split in enumerate(splits):
|
|
360
369
|
meta = deepcopy(doc.meta) if doc.meta else {}
|
|
361
|
-
chunk_start_page =
|
|
362
|
-
|
|
370
|
+
chunk_start_page = (
|
|
371
|
+
document_start_page
|
|
372
|
+
+ doc.content.count(self.page_break_character, 0, split["source_start_idx"])
|
|
373
|
+
# breaks the chunk opens with end the previous page, so its text is on a later one
|
|
374
|
+
+ _leading_page_breaks(split["content"], self.page_break_character)
|
|
363
375
|
)
|
|
364
376
|
meta.update({"source_id": doc.id, "page_number": chunk_start_page, "split_id": split_idx})
|
|
365
377
|
from_header_split = split["from_header_split"]
|
|
@@ -386,7 +398,8 @@ class MarkdownHeaderSplitter:
|
|
|
386
398
|
:returns: A dictionary with the following key:
|
|
387
399
|
- `documents`: List of documents with the split texts. Each document includes:
|
|
388
400
|
- A metadata field `source_id` to track the original document.
|
|
389
|
-
- A metadata field `page_number`
|
|
401
|
+
- A metadata field `page_number` with the page the chunk starts on, counting
|
|
402
|
+
`page_break_character` occurrences in the original document.
|
|
390
403
|
- A metadata field `split_id` to identify the split chunk index within its parent document.
|
|
391
404
|
- All other metadata copied from the original document.
|
|
392
405
|
:raises ValueError: If a document has `None` content.
|
|
@@ -7,6 +7,7 @@ from copy import deepcopy
|
|
|
7
7
|
from typing import Any, Literal
|
|
8
8
|
|
|
9
9
|
from haystack import Document, component, default_to_dict, logging
|
|
10
|
+
from haystack.components.preprocessors._page_numbers import _leading_page_breaks
|
|
10
11
|
from haystack.lazy_imports import LazyImport
|
|
11
12
|
|
|
12
13
|
with LazyImport("Run 'pip install tiktoken'") as tiktoken_imports:
|
|
@@ -82,6 +83,8 @@ class RecursiveDocumentSplitter:
|
|
|
82
83
|
If no separators are provided, the default separators ["\\n\\n", "sentence", "\\n", " "] are used.
|
|
83
84
|
:param sentence_splitter_params: Optional parameters to pass to the sentence tokenizer.
|
|
84
85
|
See: haystack.components.preprocessors.sentence_tokenizer.SentenceSplitter for more information.
|
|
86
|
+
The chunks keep the whitespace between sentences whether `keep_white_spaces` is True or False; it only
|
|
87
|
+
changes the rules the tokenizer uses to decide where a sentence ends.
|
|
85
88
|
|
|
86
89
|
:raises ValueError: If the overlap is greater than or equal to the chunk size or if the overlap is negative, or
|
|
87
90
|
if any separator is not a string.
|
|
@@ -151,11 +154,6 @@ class RecursiveDocumentSplitter:
|
|
|
151
154
|
:returns:
|
|
152
155
|
A tuple containing the current chunk and the remaining chunk.
|
|
153
156
|
"""
|
|
154
|
-
if self.split_units == "word":
|
|
155
|
-
words = current_chunk.split()
|
|
156
|
-
current_chunk = " ".join(words[: self.split_length])
|
|
157
|
-
remaining_words = words[self.split_length :]
|
|
158
|
-
return current_chunk, " ".join(remaining_words)
|
|
159
157
|
if self.split_units == "char":
|
|
160
158
|
text = current_chunk
|
|
161
159
|
current_chunk = text[: self.split_length]
|
|
@@ -172,9 +170,10 @@ class RecursiveDocumentSplitter:
|
|
|
172
170
|
"""
|
|
173
171
|
Applies an overlap between consecutive chunks if the chunk_overlap attribute is greater than zero.
|
|
174
172
|
|
|
175
|
-
Works for
|
|
176
|
-
adds the trimmed content to the next chunk. If the last chunk is still
|
|
177
|
-
and adds the first chunk to the list. This process continues until the
|
|
173
|
+
Works for character- and token-level splitting, words are handled by `_apply_word_overlap`. It trims the last
|
|
174
|
+
chunk if it exceeds the split_length and adds the trimmed content to the next chunk. If the last chunk is still
|
|
175
|
+
too long after trimming, it splits it and adds the first chunk to the list. This process continues until the
|
|
176
|
+
last chunk is within the split_length.
|
|
178
177
|
|
|
179
178
|
:param chunks: A list of text chunks.
|
|
180
179
|
:returns:
|
|
@@ -203,9 +202,7 @@ class RecursiveDocumentSplitter:
|
|
|
203
202
|
if self._chunk_length(current_chunk) > self.split_length:
|
|
204
203
|
current_chunk, remaining_text = self._split_chunk(current_chunk)
|
|
205
204
|
if idx < len(chunks) - 1:
|
|
206
|
-
if self.split_units == "
|
|
207
|
-
chunks[idx + 1] = remaining_text + " " + chunks[idx + 1]
|
|
208
|
-
elif self.split_units == "token":
|
|
205
|
+
if self.split_units == "token":
|
|
209
206
|
# For token-based splitting, combine at token level
|
|
210
207
|
# at this point we know that the tokenizer is already initialized
|
|
211
208
|
remaining_tokens = self.tiktoken_tokenizer.encode_ordinary(remaining_text) # type: ignore
|
|
@@ -246,9 +243,7 @@ class RecursiveDocumentSplitter:
|
|
|
246
243
|
return overlapped_chunks
|
|
247
244
|
|
|
248
245
|
def _create_chunk_starting_with_overlap(self, chunk: str, overlap: str) -> str:
|
|
249
|
-
if self.split_units == "
|
|
250
|
-
current_chunk = overlap + " " + chunk
|
|
251
|
-
elif self.split_units == "token":
|
|
246
|
+
if self.split_units == "token":
|
|
252
247
|
# For token-based splitting, combine at token level
|
|
253
248
|
# at this point we know that the tokenizer is already initialized
|
|
254
249
|
overlap_tokens = self.tiktoken_tokenizer.encode_ordinary(overlap) # type: ignore
|
|
@@ -263,10 +258,7 @@ class RecursiveDocumentSplitter:
|
|
|
263
258
|
prev_chunk = overlapped_chunks[-1]
|
|
264
259
|
overlap_start = max(0, self._chunk_length(prev_chunk) - self.split_overlap)
|
|
265
260
|
|
|
266
|
-
if self.split_units == "
|
|
267
|
-
word_chunks = prev_chunk.split()
|
|
268
|
-
overlap = " ".join(word_chunks[overlap_start:])
|
|
269
|
-
elif self.split_units == "token":
|
|
261
|
+
if self.split_units == "token":
|
|
270
262
|
# For token-based splitting, handle overlap at token level
|
|
271
263
|
# at this point we know that the tokenizer is already initialized
|
|
272
264
|
tokens = self.tiktoken_tokenizer.encode_ordinary(prev_chunk) # type: ignore
|
|
@@ -277,6 +269,58 @@ class RecursiveDocumentSplitter:
|
|
|
277
269
|
|
|
278
270
|
return overlap, prev_chunk
|
|
279
271
|
|
|
272
|
+
def _apply_word_overlap(self, chunks: list[str]) -> list[tuple[int, int]]:
|
|
273
|
+
"""
|
|
274
|
+
Applies an overlap between consecutive chunks when splitting by words.
|
|
275
|
+
|
|
276
|
+
Follows `_apply_overlap`: each chunk starts with the last `split_overlap` words of the chunk before it, and a
|
|
277
|
+
chunk that grows past `split_length` words is trimmed, its remaining words moving on to the next chunk. The
|
|
278
|
+
chunks are cut out of the text rather than rebuilt from their words, so they keep its whitespace and page
|
|
279
|
+
breaks. Words are counted as in `_chunk_length`.
|
|
280
|
+
|
|
281
|
+
:param chunks: A list of text chunks, as returned by `_chunk_text`.
|
|
282
|
+
:returns:
|
|
283
|
+
The `(start, end)` character offsets of each chunk with the overlap applied, in the concatenated `chunks`.
|
|
284
|
+
"""
|
|
285
|
+
# The words of every chunk, as offsets into the concatenated chunks. A chunk of only spaces has no words and
|
|
286
|
+
# joins the chunk before it (or after it, at the start of the text).
|
|
287
|
+
words: list[tuple[int, int]] = []
|
|
288
|
+
chunk_word_ends: list[int] = []
|
|
289
|
+
chunk_starts: list[int] = []
|
|
290
|
+
offset = 0
|
|
291
|
+
for chunk in chunks:
|
|
292
|
+
chunk_words = [(offset + match.start(), offset + match.end()) for match in re.finditer(r"[^ ]+", chunk)]
|
|
293
|
+
if chunk_words:
|
|
294
|
+
words.extend(chunk_words)
|
|
295
|
+
chunk_word_ends.append(len(words))
|
|
296
|
+
chunk_starts.append(offset)
|
|
297
|
+
offset += len(chunk)
|
|
298
|
+
if not words:
|
|
299
|
+
return [(0, offset)]
|
|
300
|
+
# a chunk's text runs up to where the next one starts
|
|
301
|
+
chunk_ends = chunk_starts[1:] + [offset]
|
|
302
|
+
|
|
303
|
+
spans = [(0, chunk_ends[0])]
|
|
304
|
+
# the words the latest chunk spans, as indices into `words`
|
|
305
|
+
first_word, end_word = 0, chunk_word_ends[0]
|
|
306
|
+
for idx in range(1, len(chunk_word_ends)):
|
|
307
|
+
if end_word - first_word <= self.split_overlap:
|
|
308
|
+
logger.warning(
|
|
309
|
+
"Overlap is the same as the previous chunk. "
|
|
310
|
+
"Consider increasing the `split_length` parameter or decreasing the `split_overlap` parameter."
|
|
311
|
+
)
|
|
312
|
+
# only the last chunk emits more than one chunk; the others hand their remaining words to the next one
|
|
313
|
+
while True:
|
|
314
|
+
first_word = max(first_word, end_word - self.split_overlap)
|
|
315
|
+
end_word = min(first_word + self.split_length, chunk_word_ends[idx])
|
|
316
|
+
# a trimmed chunk ends after its last word, and the next chunk picks up from its overlap
|
|
317
|
+
end = chunk_ends[idx] if end_word == chunk_word_ends[idx] else words[end_word - 1][1]
|
|
318
|
+
spans.append((words[first_word][0], end))
|
|
319
|
+
if idx < len(chunk_word_ends) - 1 or end_word == chunk_word_ends[idx]:
|
|
320
|
+
break
|
|
321
|
+
|
|
322
|
+
return spans
|
|
323
|
+
|
|
280
324
|
def _chunk_length(self, text: str) -> int:
|
|
281
325
|
"""
|
|
282
326
|
Get the length of the chunk in the specified units (words, characters, or tokens).
|
|
@@ -310,10 +354,19 @@ class RecursiveDocumentSplitter:
|
|
|
310
354
|
return [text]
|
|
311
355
|
|
|
312
356
|
for curr_separator in self.separators:
|
|
357
|
+
# text after the last split that the splits leave out
|
|
358
|
+
trailing_text = ""
|
|
313
359
|
if curr_separator == "sentence":
|
|
314
360
|
# re. ignore: correct SentenceSplitter initialization is checked at the initialization of the component
|
|
315
361
|
sentence_with_spans = self.nltk_tokenizer.split_sentences(text) # type: ignore
|
|
316
|
-
|
|
362
|
+
# Each split runs from where its sentence starts to where the next one does, so the splits keep the
|
|
363
|
+
# whitespace between sentences even where the tokenizer leaves it out (keep_white_spaces=False).
|
|
364
|
+
# Leaving it out would glue sentences together, lose page breaks and shift every later chunk's offset.
|
|
365
|
+
split_bounds = [sentence["start"] for sentence in sentence_with_spans]
|
|
366
|
+
if split_bounds:
|
|
367
|
+
split_bounds.append(sentence_with_spans[-1]["end"])
|
|
368
|
+
trailing_text = text[split_bounds[-1] :]
|
|
369
|
+
splits = [text[start:end] for start, end in zip(split_bounds, split_bounds[1:], strict=False)]
|
|
317
370
|
else:
|
|
318
371
|
# add escape "\" to the separator and wrapped it in a group so that it's included in the splits as well
|
|
319
372
|
escaped_separator = re.escape(curr_separator)
|
|
@@ -368,6 +421,9 @@ class RecursiveDocumentSplitter:
|
|
|
368
421
|
chunks.append("".join(current_chunk))
|
|
369
422
|
|
|
370
423
|
if chunks:
|
|
424
|
+
# like the whitespace left over by _fall_back_to_fixed_chunking, attach it to the last chunk rather
|
|
425
|
+
# than making it a chunk of its own
|
|
426
|
+
chunks[-1] += trailing_text
|
|
371
427
|
return chunks
|
|
372
428
|
|
|
373
429
|
# if no separator worked, fall back to word- or character-level chunking
|
|
@@ -438,20 +494,34 @@ class RecursiveDocumentSplitter:
|
|
|
438
494
|
)
|
|
439
495
|
|
|
440
496
|
def _run_one(self, doc: Document) -> list[Document]:
|
|
441
|
-
|
|
497
|
+
content = doc.content or "" # run() skips documents without content, so this is a non-empty string
|
|
498
|
+
chunks = self._chunk_text(content)
|
|
442
499
|
chunks = chunks[:-1] if len(chunks[-1]) == 0 else chunks # remove last empty chunk if it exists
|
|
443
500
|
|
|
444
501
|
# apply the overlap once, on the fully chunked list, so that chunks produced
|
|
445
502
|
# at inner recursion levels don't get the overlap applied a second time
|
|
446
|
-
if self.split_overlap > 0:
|
|
447
|
-
|
|
448
|
-
|
|
449
|
-
|
|
450
|
-
|
|
503
|
+
if self.split_overlap > 0 and self.split_units == "word":
|
|
504
|
+
text = "".join(chunks)
|
|
505
|
+
spans = self._apply_word_overlap(chunks)
|
|
506
|
+
chunks = [text[start:end] for start, end in spans]
|
|
507
|
+
chunk_starts = [start for start, _ in spans]
|
|
508
|
+
else:
|
|
509
|
+
if self.split_overlap > 0:
|
|
510
|
+
chunks = self._apply_overlap(chunks)
|
|
511
|
+
# each chunk starts where the previous one ends, minus the overlap it repeats
|
|
512
|
+
chunk_starts = [0]
|
|
513
|
+
for chunk in chunks[:-1]:
|
|
514
|
+
overlap = self._get_overlap([chunk])[0] if self.split_overlap > 0 else ""
|
|
515
|
+
chunk_starts.append(chunk_starts[-1] + len(chunk) - len(overlap))
|
|
451
516
|
|
|
452
517
|
new_docs: list[Document] = []
|
|
518
|
+
# Chunk starts never decrease, so the page breaks before each chunk are counted on from the previous one
|
|
519
|
+
# rather than from the start of the document every time.
|
|
520
|
+
page_breaks_before = 0
|
|
453
521
|
|
|
454
|
-
for split_nr, chunk in enumerate(chunks):
|
|
522
|
+
for split_nr, (chunk, current_position) in enumerate(zip(chunks, chunk_starts, strict=True)):
|
|
523
|
+
if split_nr > 0:
|
|
524
|
+
page_breaks_before += content.count("\f", chunk_starts[split_nr - 1], current_position)
|
|
455
525
|
meta = deepcopy(doc.meta)
|
|
456
526
|
meta["source_id"] = doc.id
|
|
457
527
|
meta["parent_id"] = doc.id
|
|
@@ -464,29 +534,11 @@ class RecursiveDocumentSplitter:
|
|
|
464
534
|
if split_nr > 0 and self.split_overlap > 0:
|
|
465
535
|
self._add_overlap_info(current_position, new_doc, new_docs)
|
|
466
536
|
|
|
467
|
-
#
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
# if there are consecutive page breaks at the end with no more text, adjust the page number
|
|
471
|
-
# e.g: "text\f\f\f" -> 3 page breaks, but current_page should be 1
|
|
472
|
-
consecutive_page_breaks = len(chunk) - len(chunk.rstrip("\f"))
|
|
537
|
+
# The page the chunk's first non-page-break character is on: breaks before the chunk, plus the
|
|
538
|
+
# ones it opens with.
|
|
539
|
+
new_doc.meta["page_number"] = 1 + page_breaks_before + _leading_page_breaks(chunk)
|
|
473
540
|
|
|
474
|
-
if consecutive_page_breaks > 0:
|
|
475
|
-
new_doc.meta["page_number"] = current_page - consecutive_page_breaks
|
|
476
|
-
else:
|
|
477
|
-
new_doc.meta["page_number"] = current_page
|
|
478
|
-
|
|
479
|
-
# keep the new chunk doc and update the current position
|
|
480
541
|
new_docs.append(new_doc)
|
|
481
|
-
# Advance current_position by chunk length minus overlap.
|
|
482
|
-
# split_overlap is in split_units, not chars, so get the actual
|
|
483
|
-
# overlap string from _get_overlap() and use its char length.
|
|
484
|
-
if self.split_overlap > 0 and split_nr < len(chunks) - 1:
|
|
485
|
-
overlap_str, _ = self._get_overlap([doc.content for doc in new_docs]) # type: ignore[misc]
|
|
486
|
-
overlap_char_len = len(overlap_str)
|
|
487
|
-
else:
|
|
488
|
-
overlap_char_len = 0
|
|
489
|
-
current_position += len(chunk) - overlap_char_len
|
|
490
542
|
|
|
491
543
|
return new_docs
|
|
492
544
|
|
|
@@ -498,7 +550,8 @@ class RecursiveDocumentSplitter:
|
|
|
498
550
|
:param documents: List of Documents to split.
|
|
499
551
|
:returns:
|
|
500
552
|
A dictionary containing a key "documents" with a List of Documents with smaller chunks of text corresponding
|
|
501
|
-
to the input documents.
|
|
553
|
+
to the input documents. Each chunk carries a metadata field `page_number` with the page the chunk
|
|
554
|
+
starts on, counting form feed ("\f") characters in the original document.
|
|
502
555
|
"""
|
|
503
556
|
if not self._is_warmed_up and ("sentence" in self.separators or self.split_units == "token"):
|
|
504
557
|
self.warm_up()
|
|
@@ -253,4 +253,4 @@ class SentenceSplitter:
|
|
|
253
253
|
logger.warning("No abbreviations file found for {language}. Using default abbreviations.", language=lang)
|
|
254
254
|
return []
|
|
255
255
|
|
|
256
|
-
return abbreviations_file.read_text().split("\n")
|
|
256
|
+
return abbreviations_file.read_text(encoding="utf-8").split("\n")
|
|
@@ -89,6 +89,10 @@ class LostInTheMiddleRanker:
|
|
|
89
89
|
if not documents:
|
|
90
90
|
return {"documents": []}
|
|
91
91
|
|
|
92
|
+
# Raise an error if any document is not textual
|
|
93
|
+
if any(doc.content is None for doc in documents):
|
|
94
|
+
raise ValueError("Some provided documents are not textual; LostInTheMiddleRanker can process only text.")
|
|
95
|
+
|
|
92
96
|
top_k = top_k or self.top_k
|
|
93
97
|
word_count_threshold = word_count_threshold or self.word_count_threshold
|
|
94
98
|
|
|
@@ -99,10 +103,6 @@ class LostInTheMiddleRanker:
|
|
|
99
103
|
if len(documents_to_reorder) == 1:
|
|
100
104
|
return {"documents": documents_to_reorder}
|
|
101
105
|
|
|
102
|
-
# Raise an error if any document is not textual
|
|
103
|
-
if any(doc.content is None for doc in documents_to_reorder):
|
|
104
|
-
raise ValueError("Some provided documents are not textual; LostInTheMiddleRanker can process only text.")
|
|
105
|
-
|
|
106
106
|
# Initialize word count and indices for the "lost in the middle" order
|
|
107
107
|
word_count = 0
|
|
108
108
|
document_index = list(range(len(documents_to_reorder)))
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/telemetry/_telemetry.py
RENAMED
|
@@ -89,7 +89,7 @@ class Telemetry:
|
|
|
89
89
|
CONFIG_PATH.parents[0].mkdir(parents=True, exist_ok=True)
|
|
90
90
|
self.user_id = str(uuid.uuid4())
|
|
91
91
|
try:
|
|
92
|
-
with open(CONFIG_PATH, "w") as outfile:
|
|
92
|
+
with open(CONFIG_PATH, "w", encoding="utf-8") as outfile:
|
|
93
93
|
yaml.dump({"user_id": self.user_id}, outfile, default_flow_style=False)
|
|
94
94
|
except Exception as e:
|
|
95
95
|
logger.debug(
|
haystack_ai-3.3.0rc1/VERSION.txt
DELETED
|
@@ -1 +0,0 @@
|
|
|
1
|
-
3.3.0-rc1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/__init__.py
RENAMED
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/__init__.py
RENAMED
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/agent.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/utils.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/caching/__init__.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/csv.py
RENAMED
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/docx.py
RENAMED
|
File without changes
|
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/html.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/json.py
RENAMED
|
File without changes
|
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/msg.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/pptx.py
RENAMED
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/pypdf.py
RENAMED
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/txt.py
RENAMED
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/utils.py
RENAMED
|
File without changes
|
{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/xlsx.py
RENAMED
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|