haystack-ai 3.3.0rc1__tar.gz → 3.4.0.dev20261002000327__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (299) hide show
  1. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/PKG-INFO +1 -1
  2. haystack_ai-3.4.0.dev20261002000327/VERSION.txt +1 -0
  3. haystack_ai-3.4.0.dev20261002000327/haystack/components/preprocessors/_page_numbers.py +38 -0
  4. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/document_splitter.py +4 -2
  5. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/embedding_based_document_splitter.py +8 -6
  6. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/markdown_header_splitter.py +19 -6
  7. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/recursive_splitter.py +101 -48
  8. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/sentence_tokenizer.py +1 -1
  9. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/lost_in_the_middle.py +4 -4
  10. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/telemetry/_telemetry.py +1 -1
  11. haystack_ai-3.3.0rc1/VERSION.txt +0 -1
  12. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/.gitignore +0 -0
  13. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/LICENSE +0 -0
  14. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/README.md +0 -0
  15. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/__init__.py +0 -0
  16. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/__init__.py +0 -0
  17. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/__init__.py +0 -0
  18. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/agent.py +0 -0
  19. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/state/__init__.py +0 -0
  20. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/state/state.py +0 -0
  21. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/state/state_utils.py +0 -0
  22. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/tool_calling.py +0 -0
  23. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/agents/utils.py +0 -0
  24. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/builders/__init__.py +0 -0
  25. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/builders/answer_builder.py +0 -0
  26. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/builders/chat_prompt_builder.py +0 -0
  27. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/builders/prompt_builder.py +0 -0
  28. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/caching/__init__.py +0 -0
  29. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/caching/cache_checker.py +0 -0
  30. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/__init__.py +0 -0
  31. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/csv.py +0 -0
  32. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/docx.py +0 -0
  33. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/file_to_file_content.py +0 -0
  34. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/html.py +0 -0
  35. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/__init__.py +0 -0
  36. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/document_to_image.py +0 -0
  37. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/file_to_document.py +0 -0
  38. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/file_to_image.py +0 -0
  39. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/image_utils.py +0 -0
  40. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/image/pdf_to_image.py +0 -0
  41. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/json.py +0 -0
  42. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/markdown.py +0 -0
  43. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/msg.py +0 -0
  44. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/multi_file_converter.py +0 -0
  45. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/output_adapter.py +0 -0
  46. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/pdfminer.py +0 -0
  47. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/pptx.py +0 -0
  48. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/pypdf.py +0 -0
  49. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/txt.py +0 -0
  50. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/utils.py +0 -0
  51. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/converters/xlsx.py +0 -0
  52. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/__init__.py +0 -0
  53. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/azure_document_embedder.py +0 -0
  54. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/azure_text_embedder.py +0 -0
  55. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/mock_document_embedder.py +0 -0
  56. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/mock_text_embedder.py +0 -0
  57. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/mock_utils.py +0 -0
  58. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/openai_document_embedder.py +0 -0
  59. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/openai_text_embedder.py +0 -0
  60. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/types/__init__.py +0 -0
  61. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/embedders/types/protocol.py +0 -0
  62. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/__init__.py +0 -0
  63. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/answer_exact_match.py +0 -0
  64. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/context_relevance.py +0 -0
  65. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/document_map.py +0 -0
  66. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/document_mrr.py +0 -0
  67. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/document_ndcg.py +0 -0
  68. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/document_recall.py +0 -0
  69. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/faithfulness.py +0 -0
  70. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/llm_evaluator.py +0 -0
  71. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/evaluators/sas_evaluator.py +0 -0
  72. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/__init__.py +0 -0
  73. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/image/__init__.py +0 -0
  74. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/image/llm_document_content_extractor.py +0 -0
  75. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/llm_metadata_extractor.py +0 -0
  76. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/extractors/regex_text_extractor.py +0 -0
  77. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/fetchers/__init__.py +0 -0
  78. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/fetchers/link_content.py +0 -0
  79. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/__init__.py +0 -0
  80. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/__init__.py +0 -0
  81. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/azure.py +0 -0
  82. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/azure_responses.py +0 -0
  83. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/fallback.py +0 -0
  84. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/llm.py +0 -0
  85. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/mock.py +0 -0
  86. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/openai.py +0 -0
  87. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/openai_responses.py +0 -0
  88. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/types/__init__.py +0 -0
  89. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/chat/types/protocol.py +0 -0
  90. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/openai_image_generator.py +0 -0
  91. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/generators/utils.py +0 -0
  92. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/__init__.py +0 -0
  93. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/answer_joiner.py +0 -0
  94. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/branch.py +0 -0
  95. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/document_joiner.py +0 -0
  96. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/list_joiner.py +0 -0
  97. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/joiners/string_joiner.py +0 -0
  98. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/__init__.py +0 -0
  99. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/csv_document_cleaner.py +0 -0
  100. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/csv_document_splitter.py +0 -0
  101. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/document_cleaner.py +0 -0
  102. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/document_preprocessor.py +0 -0
  103. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/hierarchical_document_splitter.py +0 -0
  104. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/python_code_splitter.py +0 -0
  105. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/preprocessors/text_cleaner.py +0 -0
  106. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/query/__init__.py +0 -0
  107. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/query/query_expander.py +0 -0
  108. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/__init__.py +0 -0
  109. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/llm_ranker.py +0 -0
  110. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/meta_field.py +0 -0
  111. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/rankers/meta_field_grouping_ranker.py +0 -0
  112. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/__init__.py +0 -0
  113. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/auto_merging_retriever.py +0 -0
  114. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/filter_retriever.py +0 -0
  115. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/in_memory/__init__.py +0 -0
  116. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/in_memory/bm25_retriever.py +0 -0
  117. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/in_memory/embedding_retriever.py +0 -0
  118. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/multi_query_embedding_retriever.py +0 -0
  119. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/multi_query_text_retriever.py +0 -0
  120. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/multi_retriever.py +0 -0
  121. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/sentence_window_retriever.py +0 -0
  122. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/text_embedding_retriever.py +0 -0
  123. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/types/__init__.py +0 -0
  124. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/retrievers/types/protocol.py +0 -0
  125. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/__init__.py +0 -0
  126. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/conditional_router.py +0 -0
  127. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/document_length_router.py +0 -0
  128. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/document_type_router.py +0 -0
  129. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/file_type_router.py +0 -0
  130. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/llm_messages_router.py +0 -0
  131. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/routers/metadata_router.py +0 -0
  132. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/samplers/__init__.py +0 -0
  133. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/samplers/top_p.py +0 -0
  134. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/validators/__init__.py +0 -0
  135. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/validators/json_schema.py +0 -0
  136. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/writers/__init__.py +0 -0
  137. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/components/writers/document_writer.py +0 -0
  138. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/__init__.py +0 -0
  139. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/component/__init__.py +0 -0
  140. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/component/component.py +0 -0
  141. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/component/sockets.py +0 -0
  142. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/component/types.py +0 -0
  143. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/errors.py +0 -0
  144. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/__init__.py +0 -0
  145. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/base.py +0 -0
  146. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/breakpoint.py +0 -0
  147. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/component_checks.py +0 -0
  148. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/descriptions.py +0 -0
  149. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/draw.py +0 -0
  150. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/pipeline.py +0 -0
  151. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/pipeline/utils.py +0 -0
  152. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/serialization.py +0 -0
  153. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/serialization_security.py +0 -0
  154. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/super_component/__init__.py +0 -0
  155. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/super_component/super_component.py +0 -0
  156. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/super_component/utils.py +0 -0
  157. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/core/type_utils.py +0 -0
  158. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/data/abbreviations/de.txt +0 -0
  159. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/data/abbreviations/en.txt +0 -0
  160. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/__init__.py +0 -0
  161. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/answer.py +0 -0
  162. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/breakpoints.py +0 -0
  163. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/byte_stream.py +0 -0
  164. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/chat_message.py +0 -0
  165. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/document.py +0 -0
  166. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/file_content.py +0 -0
  167. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/image_content.py +0 -0
  168. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/skill_info.py +0 -0
  169. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/sparse_embedding.py +0 -0
  170. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/dataclasses/streaming_chunk.py +0 -0
  171. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/__init__.py +0 -0
  172. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/errors/__init__.py +0 -0
  173. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/errors/errors.py +0 -0
  174. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/in_memory/__init__.py +0 -0
  175. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/in_memory/document_store.py +0 -0
  176. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/types/__init__.py +0 -0
  177. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/types/filter_policy.py +0 -0
  178. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/types/policy.py +0 -0
  179. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/document_stores/types/protocol.py +0 -0
  180. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/errors.py +0 -0
  181. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/evaluation/__init__.py +0 -0
  182. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/evaluation/eval_run_result.py +0 -0
  183. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/__init__.py +0 -0
  184. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/budget/__init__.py +0 -0
  185. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/budget/hooks.py +0 -0
  186. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/AGENTS.md +0 -0
  187. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/CLAUDE.md +0 -0
  188. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/__init__.py +0 -0
  189. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/hooks.py +0 -0
  190. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/sliding_window.py +0 -0
  191. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/summarization.py +0 -0
  192. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/tool_result_pruning.py +0 -0
  193. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/types/__init__.py +0 -0
  194. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/types/protocol.py +0 -0
  195. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/compaction/utils.py +0 -0
  196. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/from_function.py +0 -0
  197. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/__init__.py +0 -0
  198. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/dataclasses.py +0 -0
  199. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/hooks.py +0 -0
  200. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/policies.py +0 -0
  201. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/strategies.py +0 -0
  202. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/types/__init__.py +0 -0
  203. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/types/protocol.py +0 -0
  204. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/human_in_the_loop/user_interfaces.py +0 -0
  205. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/invocation.py +0 -0
  206. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/protocol.py +0 -0
  207. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/__init__.py +0 -0
  208. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/hooks.py +0 -0
  209. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/policies.py +0 -0
  210. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/stores.py +0 -0
  211. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/types/__init__.py +0 -0
  212. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/tool_result_offloading/types/protocol.py +0 -0
  213. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/hooks/utils.py +0 -0
  214. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/lazy_imports.py +0 -0
  215. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/logging.py +0 -0
  216. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/marshal/__init__.py +0 -0
  217. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/marshal/protocol.py +0 -0
  218. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/marshal/yaml.py +0 -0
  219. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/py.typed +0 -0
  220. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/__init__.py +0 -0
  221. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/file_system/__init__.py +0 -0
  222. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/file_system/skill_store.py +0 -0
  223. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/types/__init__.py +0 -0
  224. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/skill_stores/types/protocol.py +0 -0
  225. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/telemetry/__init__.py +0 -0
  226. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/telemetry/_environment.py +0 -0
  227. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/__init__.py +0 -0
  228. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/callable_serialization/random_callable.py +0 -0
  229. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/document_store.py +0 -0
  230. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/document_store_async.py +0 -0
  231. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/factory.py +0 -0
  232. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/__init__.py +0 -0
  233. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/accumulate.py +0 -0
  234. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/add_value.py +0 -0
  235. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/concatenate.py +0 -0
  236. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/double.py +0 -0
  237. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/fstring.py +0 -0
  238. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/future_annotations.py +0 -0
  239. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/greet.py +0 -0
  240. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/hello.py +0 -0
  241. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/joiner.py +0 -0
  242. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/parity.py +0 -0
  243. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/remainder.py +0 -0
  244. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/repeat.py +0 -0
  245. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/subtract.py +0 -0
  246. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/sum.py +0 -0
  247. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/text_splitter.py +0 -0
  248. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/sample_components/threshold.py +0 -0
  249. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/telemetry.py +0 -0
  250. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/testing/test_utils.py +0 -0
  251. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/__init__.py +0 -0
  252. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/approximate_counter.py +0 -0
  253. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/openai_counter.py +0 -0
  254. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/tiktoken_counter.py +0 -0
  255. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/types/__init__.py +0 -0
  256. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/types/protocol.py +0 -0
  257. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/token_counters/utils.py +0 -0
  258. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/__init__.py +0 -0
  259. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/agent_tool.py +0 -0
  260. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/component_tool.py +0 -0
  261. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/errors.py +0 -0
  262. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/from_function.py +0 -0
  263. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/parameters_schema_utils.py +0 -0
  264. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/pipeline_tool.py +0 -0
  265. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/searchable_toolset.py +0 -0
  266. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/serde_utils.py +0 -0
  267. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/skills/__init__.py +0 -0
  268. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/skills/skill_toolset.py +0 -0
  269. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/tool.py +0 -0
  270. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/tool_types.py +0 -0
  271. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/toolset.py +0 -0
  272. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tools/utils.py +0 -0
  273. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tracing/__init__.py +0 -0
  274. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tracing/logging_tracer.py +0 -0
  275. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tracing/tracer.py +0 -0
  276. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/tracing/utils.py +0 -0
  277. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/__init__.py +0 -0
  278. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/async_utils.py +0 -0
  279. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/auth.py +0 -0
  280. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/azure.py +0 -0
  281. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/base_serialization.py +0 -0
  282. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/callable_serialization.py +0 -0
  283. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/dataclasses.py +0 -0
  284. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/deserialization.py +0 -0
  285. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/device.py +0 -0
  286. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/experimental.py +0 -0
  287. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/filters.py +0 -0
  288. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/hf.py +0 -0
  289. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/http_client.py +0 -0
  290. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/jinja2_chat_extension.py +0 -0
  291. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/jinja2_extensions.py +0 -0
  292. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/jinja2_sandbox.py +0 -0
  293. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/jupyter.py +0 -0
  294. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/misc.py +0 -0
  295. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/requests_utils.py +0 -0
  296. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/type_serialization.py +0 -0
  297. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/utils/url_validation.py +0 -0
  298. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/haystack/version.py +0 -0
  299. {haystack_ai-3.3.0rc1 → haystack_ai-3.4.0.dev20261002000327}/pyproject.toml +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: haystack-ai
3
- Version: 3.3.0-rc1
3
+ Version: 3.4.0.dev20261002000327
4
4
  Summary: LLM framework to build customizable, production-ready LLM applications. Connect components (models, vector DBs, file converters) to pipelines or agents that can interact with your data.
5
5
  Project-URL: CI: GitHub, https://github.com/deepset-ai/haystack/actions
6
6
  Project-URL: Docs: RTD, https://haystack.deepset.ai/overview/intro
@@ -0,0 +1 @@
1
+ 3.4.0.dev20261002000327
@@ -0,0 +1,38 @@
1
+ # SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
2
+ #
3
+ # SPDX-License-Identifier: Apache-2.0
4
+
5
+ """
6
+ The `page_number` convention shared by the splitters.
7
+
8
+ Splitters record the page a chunk came from in its `page_number` metadata field, counting page breaks
9
+ (form feed, "\f") in the source document. The page reported is the one the chunk's first non-page-break
10
+ character lies on: a chunk that spans a page break belongs to the page it starts on, and page breaks a
11
+ chunk opens with belong to that chunk's own page rather than to the page it was carved out of.
12
+ """
13
+
14
+
15
+ def _leading_page_breaks(text: str, page_break_character: str = "\f") -> int:
16
+ """
17
+ Count how many pages a chunk's text starts past the page its first character is on.
18
+
19
+ This is the run of page breaks the chunk opens with, except for a chunk made up entirely of page
20
+ breaks: that chunk is an empty page and stays on the page it starts on, rather than being pushed onto
21
+ the following one. `split_by="page"` emits such chunks for every blank page.
22
+
23
+ :param text: The chunk to inspect.
24
+ :param page_break_character: The character sequence marking a page break.
25
+ :returns: The number of pages to advance, 0 if the chunk starts with anything but a page break or
26
+ consists only of page breaks.
27
+ """
28
+ if not page_break_character:
29
+ return 0
30
+
31
+ offset = 0
32
+ while text.startswith(page_break_character, offset):
33
+ offset += len(page_break_character)
34
+
35
+ # Nothing but page breaks: an empty page, which belongs to the page it starts on.
36
+ if offset >= len(text):
37
+ return 0
38
+ return offset // len(page_break_character)
@@ -9,6 +9,7 @@ from typing import Any, Literal
9
9
  from more_itertools import windowed
10
10
 
11
11
  from haystack import Document, component, logging
12
+ from haystack.components.preprocessors._page_numbers import _leading_page_breaks
12
13
  from haystack.components.preprocessors.sentence_tokenizer import Language, SentenceSplitter, nltk_imports
13
14
  from haystack.core.serialization import default_from_dict, default_to_dict
14
15
  from haystack.lazy_imports import LazyImport
@@ -199,7 +200,8 @@ class DocumentSplitter:
199
200
  :returns: A dictionary with the following key:
200
201
  - `documents`: List of documents with the split texts. Each document includes:
201
202
  - A metadata field `source_id` to track the original document.
202
- - A metadata field `page_number` to track the original page number.
203
+ - A metadata field `page_number` with the page the chunk starts on, counting form feed
204
+ ("\f") characters in the original document.
203
205
  - All other metadata copied from the original document.
204
206
 
205
207
  :raises TypeError: if the input is not a list of Documents.
@@ -450,7 +452,7 @@ class DocumentSplitter:
450
452
 
451
453
  for i, (txt, split_idx) in enumerate(zip(text_splits, splits_start_idxs, strict=True)):
452
454
  copied_meta = deepcopy(meta)
453
- copied_meta["page_number"] = splits_pages[i]
455
+ copied_meta["page_number"] = splits_pages[i] + _leading_page_breaks(txt)
454
456
  copied_meta["split_id"] = i
455
457
  copied_meta["split_idx_start"] = split_idx
456
458
  doc = Document(content=txt, meta=copied_meta)
@@ -12,6 +12,7 @@ import numpy as np
12
12
 
13
13
  from haystack import Document, component, logging
14
14
  from haystack.components.embedders.types import DocumentEmbedder
15
+ from haystack.components.preprocessors._page_numbers import _leading_page_breaks
15
16
  from haystack.components.preprocessors.sentence_tokenizer import Language, SentenceSplitter
16
17
  from haystack.core.serialization import component_to_dict, default_from_dict, default_to_dict
17
18
  from haystack.utils.async_utils import _execute_component_async
@@ -179,7 +180,8 @@ class EmbeddingBasedDocumentSplitter:
179
180
  - A metadata field `source_id` to track the original document.
180
181
  - A metadata field `split_id` to track the split number.
181
182
  - A metadata field `split_idx_start` with the character offset of the chunk in the original document.
182
- - A metadata field `page_number` to track the original page number.
183
+ - A metadata field `page_number` with the page the chunk starts on, counting form feed
184
+ ("\f") characters in the original document.
183
185
  - All other metadata copied from the original document.
184
186
 
185
187
  :raises RuntimeError: If the component wasn't warmed up.
@@ -220,7 +222,8 @@ class EmbeddingBasedDocumentSplitter:
220
222
  - A metadata field `source_id` to track the original document.
221
223
  - A metadata field `split_id` to track the split number.
222
224
  - A metadata field `split_idx_start` with the character offset of the chunk in the original document.
223
- - A metadata field `page_number` to track the original page number.
225
+ - A metadata field `page_number` with the page the chunk starts on, counting form feed
226
+ ("\f") characters in the original document.
224
227
  - All other metadata copied from the original document.
225
228
 
226
229
  :raises RuntimeError: If the component wasn't warmed up.
@@ -549,12 +552,11 @@ class EmbeddingBasedDocumentSplitter:
549
552
  split_meta["split_id"] = i
550
553
  split_meta["split_idx_start"] = current_char_pos
551
554
 
552
- # Calculate page number for this split
553
- # Count page breaks in the split itself
554
555
  page_breaks_in_split = split_text.count("\f")
555
556
 
556
- # Calculate the page number for this split
557
- split_meta["page_number"] = current_page
557
+ # current_page is the page this split's first character is on; a split that opens with page
558
+ # breaks starts its text on a later page.
559
+ split_meta["page_number"] = current_page + _leading_page_breaks(split_text)
558
560
 
559
561
  doc = Document(content=split_text, meta=split_meta)
560
562
  documents.append(doc)
@@ -8,6 +8,7 @@ from typing import Literal
8
8
 
9
9
  from haystack import Document, component, logging
10
10
  from haystack.components.preprocessors import DocumentSplitter
11
+ from haystack.components.preprocessors._page_numbers import _leading_page_breaks
11
12
 
12
13
  logger = logging.getLogger(__name__)
13
14
 
@@ -294,8 +295,12 @@ class MarkdownHeaderSplitter:
294
295
  secondary_content_start_idx = header_match.end()
295
296
  content_for_splitting = doc.content[secondary_content_start_idx:]
296
297
 
297
- # The page this header chunk starts on; its splits are numbered relative to it.
298
- chunk_start_page = doc.meta.get("page_number", 1)
298
+ # The page this header chunk's first character is on; its splits are numbered relative to it.
299
+ # doc.meta holds the page the chunk's *text* starts on, which already counts the breaks the chunk
300
+ # opens with, and page_break_ends below counts those same breaks again - so take them back off.
301
+ chunk_start_page = doc.meta.get("page_number", 1) - _leading_page_breaks(
302
+ doc.content, self.page_break_character
303
+ )
299
304
 
300
305
  clean_meta = {k: v for k, v in doc.meta.items() if k != "split_id"}
301
306
 
@@ -312,7 +317,11 @@ class MarkdownHeaderSplitter:
312
317
  while page_break_count < len(page_break_ends) and page_break_ends[page_break_count] <= split_start_idx:
313
318
  page_break_count += 1
314
319
 
315
- split.meta["page_number"] = chunk_start_page + page_break_count
320
+ split.meta["page_number"] = (
321
+ chunk_start_page
322
+ + page_break_count
323
+ + _leading_page_breaks(split.content or "", self.page_break_character)
324
+ )
316
325
  split.meta["split_id"] = current_split_id
317
326
  if "source_id" in doc.meta:
318
327
  split.meta["source_id"] = doc.meta["source_id"]
@@ -358,8 +367,11 @@ class MarkdownHeaderSplitter:
358
367
  )
359
368
  for split_idx, split in enumerate(splits):
360
369
  meta = deepcopy(doc.meta) if doc.meta else {}
361
- chunk_start_page = document_start_page + doc.content.count(
362
- self.page_break_character, 0, split["source_start_idx"]
370
+ chunk_start_page = (
371
+ document_start_page
372
+ + doc.content.count(self.page_break_character, 0, split["source_start_idx"])
373
+ # breaks the chunk opens with end the previous page, so its text is on a later one
374
+ + _leading_page_breaks(split["content"], self.page_break_character)
363
375
  )
364
376
  meta.update({"source_id": doc.id, "page_number": chunk_start_page, "split_id": split_idx})
365
377
  from_header_split = split["from_header_split"]
@@ -386,7 +398,8 @@ class MarkdownHeaderSplitter:
386
398
  :returns: A dictionary with the following key:
387
399
  - `documents`: List of documents with the split texts. Each document includes:
388
400
  - A metadata field `source_id` to track the original document.
389
- - A metadata field `page_number` to track the original page number.
401
+ - A metadata field `page_number` with the page the chunk starts on, counting
402
+ `page_break_character` occurrences in the original document.
390
403
  - A metadata field `split_id` to identify the split chunk index within its parent document.
391
404
  - All other metadata copied from the original document.
392
405
  :raises ValueError: If a document has `None` content.
@@ -7,6 +7,7 @@ from copy import deepcopy
7
7
  from typing import Any, Literal
8
8
 
9
9
  from haystack import Document, component, default_to_dict, logging
10
+ from haystack.components.preprocessors._page_numbers import _leading_page_breaks
10
11
  from haystack.lazy_imports import LazyImport
11
12
 
12
13
  with LazyImport("Run 'pip install tiktoken'") as tiktoken_imports:
@@ -82,6 +83,8 @@ class RecursiveDocumentSplitter:
82
83
  If no separators are provided, the default separators ["\\n\\n", "sentence", "\\n", " "] are used.
83
84
  :param sentence_splitter_params: Optional parameters to pass to the sentence tokenizer.
84
85
  See: haystack.components.preprocessors.sentence_tokenizer.SentenceSplitter for more information.
86
+ The chunks keep the whitespace between sentences whether `keep_white_spaces` is True or False; it only
87
+ changes the rules the tokenizer uses to decide where a sentence ends.
85
88
 
86
89
  :raises ValueError: If the overlap is greater than or equal to the chunk size or if the overlap is negative, or
87
90
  if any separator is not a string.
@@ -151,11 +154,6 @@ class RecursiveDocumentSplitter:
151
154
  :returns:
152
155
  A tuple containing the current chunk and the remaining chunk.
153
156
  """
154
- if self.split_units == "word":
155
- words = current_chunk.split()
156
- current_chunk = " ".join(words[: self.split_length])
157
- remaining_words = words[self.split_length :]
158
- return current_chunk, " ".join(remaining_words)
159
157
  if self.split_units == "char":
160
158
  text = current_chunk
161
159
  current_chunk = text[: self.split_length]
@@ -172,9 +170,10 @@ class RecursiveDocumentSplitter:
172
170
  """
173
171
  Applies an overlap between consecutive chunks if the chunk_overlap attribute is greater than zero.
174
172
 
175
- Works for both word- and character-level splitting. It trims the last chunk if it exceeds the split_length and
176
- adds the trimmed content to the next chunk. If the last chunk is still too long after trimming, it splits it
177
- and adds the first chunk to the list. This process continues until the last chunk is within the split_length.
173
+ Works for character- and token-level splitting, words are handled by `_apply_word_overlap`. It trims the last
174
+ chunk if it exceeds the split_length and adds the trimmed content to the next chunk. If the last chunk is still
175
+ too long after trimming, it splits it and adds the first chunk to the list. This process continues until the
176
+ last chunk is within the split_length.
178
177
 
179
178
  :param chunks: A list of text chunks.
180
179
  :returns:
@@ -203,9 +202,7 @@ class RecursiveDocumentSplitter:
203
202
  if self._chunk_length(current_chunk) > self.split_length:
204
203
  current_chunk, remaining_text = self._split_chunk(current_chunk)
205
204
  if idx < len(chunks) - 1:
206
- if self.split_units == "word":
207
- chunks[idx + 1] = remaining_text + " " + chunks[idx + 1]
208
- elif self.split_units == "token":
205
+ if self.split_units == "token":
209
206
  # For token-based splitting, combine at token level
210
207
  # at this point we know that the tokenizer is already initialized
211
208
  remaining_tokens = self.tiktoken_tokenizer.encode_ordinary(remaining_text) # type: ignore
@@ -246,9 +243,7 @@ class RecursiveDocumentSplitter:
246
243
  return overlapped_chunks
247
244
 
248
245
  def _create_chunk_starting_with_overlap(self, chunk: str, overlap: str) -> str:
249
- if self.split_units == "word":
250
- current_chunk = overlap + " " + chunk
251
- elif self.split_units == "token":
246
+ if self.split_units == "token":
252
247
  # For token-based splitting, combine at token level
253
248
  # at this point we know that the tokenizer is already initialized
254
249
  overlap_tokens = self.tiktoken_tokenizer.encode_ordinary(overlap) # type: ignore
@@ -263,10 +258,7 @@ class RecursiveDocumentSplitter:
263
258
  prev_chunk = overlapped_chunks[-1]
264
259
  overlap_start = max(0, self._chunk_length(prev_chunk) - self.split_overlap)
265
260
 
266
- if self.split_units == "word":
267
- word_chunks = prev_chunk.split()
268
- overlap = " ".join(word_chunks[overlap_start:])
269
- elif self.split_units == "token":
261
+ if self.split_units == "token":
270
262
  # For token-based splitting, handle overlap at token level
271
263
  # at this point we know that the tokenizer is already initialized
272
264
  tokens = self.tiktoken_tokenizer.encode_ordinary(prev_chunk) # type: ignore
@@ -277,6 +269,58 @@ class RecursiveDocumentSplitter:
277
269
 
278
270
  return overlap, prev_chunk
279
271
 
272
+ def _apply_word_overlap(self, chunks: list[str]) -> list[tuple[int, int]]:
273
+ """
274
+ Applies an overlap between consecutive chunks when splitting by words.
275
+
276
+ Follows `_apply_overlap`: each chunk starts with the last `split_overlap` words of the chunk before it, and a
277
+ chunk that grows past `split_length` words is trimmed, its remaining words moving on to the next chunk. The
278
+ chunks are cut out of the text rather than rebuilt from their words, so they keep its whitespace and page
279
+ breaks. Words are counted as in `_chunk_length`.
280
+
281
+ :param chunks: A list of text chunks, as returned by `_chunk_text`.
282
+ :returns:
283
+ The `(start, end)` character offsets of each chunk with the overlap applied, in the concatenated `chunks`.
284
+ """
285
+ # The words of every chunk, as offsets into the concatenated chunks. A chunk of only spaces has no words and
286
+ # joins the chunk before it (or after it, at the start of the text).
287
+ words: list[tuple[int, int]] = []
288
+ chunk_word_ends: list[int] = []
289
+ chunk_starts: list[int] = []
290
+ offset = 0
291
+ for chunk in chunks:
292
+ chunk_words = [(offset + match.start(), offset + match.end()) for match in re.finditer(r"[^ ]+", chunk)]
293
+ if chunk_words:
294
+ words.extend(chunk_words)
295
+ chunk_word_ends.append(len(words))
296
+ chunk_starts.append(offset)
297
+ offset += len(chunk)
298
+ if not words:
299
+ return [(0, offset)]
300
+ # a chunk's text runs up to where the next one starts
301
+ chunk_ends = chunk_starts[1:] + [offset]
302
+
303
+ spans = [(0, chunk_ends[0])]
304
+ # the words the latest chunk spans, as indices into `words`
305
+ first_word, end_word = 0, chunk_word_ends[0]
306
+ for idx in range(1, len(chunk_word_ends)):
307
+ if end_word - first_word <= self.split_overlap:
308
+ logger.warning(
309
+ "Overlap is the same as the previous chunk. "
310
+ "Consider increasing the `split_length` parameter or decreasing the `split_overlap` parameter."
311
+ )
312
+ # only the last chunk emits more than one chunk; the others hand their remaining words to the next one
313
+ while True:
314
+ first_word = max(first_word, end_word - self.split_overlap)
315
+ end_word = min(first_word + self.split_length, chunk_word_ends[idx])
316
+ # a trimmed chunk ends after its last word, and the next chunk picks up from its overlap
317
+ end = chunk_ends[idx] if end_word == chunk_word_ends[idx] else words[end_word - 1][1]
318
+ spans.append((words[first_word][0], end))
319
+ if idx < len(chunk_word_ends) - 1 or end_word == chunk_word_ends[idx]:
320
+ break
321
+
322
+ return spans
323
+
280
324
  def _chunk_length(self, text: str) -> int:
281
325
  """
282
326
  Get the length of the chunk in the specified units (words, characters, or tokens).
@@ -310,10 +354,19 @@ class RecursiveDocumentSplitter:
310
354
  return [text]
311
355
 
312
356
  for curr_separator in self.separators:
357
+ # text after the last split that the splits leave out
358
+ trailing_text = ""
313
359
  if curr_separator == "sentence":
314
360
  # re. ignore: correct SentenceSplitter initialization is checked at the initialization of the component
315
361
  sentence_with_spans = self.nltk_tokenizer.split_sentences(text) # type: ignore
316
- splits = [sentence["sentence"] for sentence in sentence_with_spans]
362
+ # Each split runs from where its sentence starts to where the next one does, so the splits keep the
363
+ # whitespace between sentences even where the tokenizer leaves it out (keep_white_spaces=False).
364
+ # Leaving it out would glue sentences together, lose page breaks and shift every later chunk's offset.
365
+ split_bounds = [sentence["start"] for sentence in sentence_with_spans]
366
+ if split_bounds:
367
+ split_bounds.append(sentence_with_spans[-1]["end"])
368
+ trailing_text = text[split_bounds[-1] :]
369
+ splits = [text[start:end] for start, end in zip(split_bounds, split_bounds[1:], strict=False)]
317
370
  else:
318
371
  # add escape "\" to the separator and wrapped it in a group so that it's included in the splits as well
319
372
  escaped_separator = re.escape(curr_separator)
@@ -368,6 +421,9 @@ class RecursiveDocumentSplitter:
368
421
  chunks.append("".join(current_chunk))
369
422
 
370
423
  if chunks:
424
+ # like the whitespace left over by _fall_back_to_fixed_chunking, attach it to the last chunk rather
425
+ # than making it a chunk of its own
426
+ chunks[-1] += trailing_text
371
427
  return chunks
372
428
 
373
429
  # if no separator worked, fall back to word- or character-level chunking
@@ -438,20 +494,34 @@ class RecursiveDocumentSplitter:
438
494
  )
439
495
 
440
496
  def _run_one(self, doc: Document) -> list[Document]:
441
- chunks = self._chunk_text(doc.content) # type: ignore # the caller already check for a non-empty doc.content
497
+ content = doc.content or "" # run() skips documents without content, so this is a non-empty string
498
+ chunks = self._chunk_text(content)
442
499
  chunks = chunks[:-1] if len(chunks[-1]) == 0 else chunks # remove last empty chunk if it exists
443
500
 
444
501
  # apply the overlap once, on the fully chunked list, so that chunks produced
445
502
  # at inner recursion levels don't get the overlap applied a second time
446
- if self.split_overlap > 0:
447
- chunks = self._apply_overlap(chunks)
448
-
449
- current_position = 0
450
- current_page = 1
503
+ if self.split_overlap > 0 and self.split_units == "word":
504
+ text = "".join(chunks)
505
+ spans = self._apply_word_overlap(chunks)
506
+ chunks = [text[start:end] for start, end in spans]
507
+ chunk_starts = [start for start, _ in spans]
508
+ else:
509
+ if self.split_overlap > 0:
510
+ chunks = self._apply_overlap(chunks)
511
+ # each chunk starts where the previous one ends, minus the overlap it repeats
512
+ chunk_starts = [0]
513
+ for chunk in chunks[:-1]:
514
+ overlap = self._get_overlap([chunk])[0] if self.split_overlap > 0 else ""
515
+ chunk_starts.append(chunk_starts[-1] + len(chunk) - len(overlap))
451
516
 
452
517
  new_docs: list[Document] = []
518
+ # Chunk starts never decrease, so the page breaks before each chunk are counted on from the previous one
519
+ # rather than from the start of the document every time.
520
+ page_breaks_before = 0
453
521
 
454
- for split_nr, chunk in enumerate(chunks):
522
+ for split_nr, (chunk, current_position) in enumerate(zip(chunks, chunk_starts, strict=True)):
523
+ if split_nr > 0:
524
+ page_breaks_before += content.count("\f", chunk_starts[split_nr - 1], current_position)
455
525
  meta = deepcopy(doc.meta)
456
526
  meta["source_id"] = doc.id
457
527
  meta["parent_id"] = doc.id
@@ -464,29 +534,11 @@ class RecursiveDocumentSplitter:
464
534
  if split_nr > 0 and self.split_overlap > 0:
465
535
  self._add_overlap_info(current_position, new_doc, new_docs)
466
536
 
467
- # count page breaks in the chunk
468
- current_page += chunk.count("\f")
469
-
470
- # if there are consecutive page breaks at the end with no more text, adjust the page number
471
- # e.g: "text\f\f\f" -> 3 page breaks, but current_page should be 1
472
- consecutive_page_breaks = len(chunk) - len(chunk.rstrip("\f"))
537
+ # The page the chunk's first non-page-break character is on: breaks before the chunk, plus the
538
+ # ones it opens with.
539
+ new_doc.meta["page_number"] = 1 + page_breaks_before + _leading_page_breaks(chunk)
473
540
 
474
- if consecutive_page_breaks > 0:
475
- new_doc.meta["page_number"] = current_page - consecutive_page_breaks
476
- else:
477
- new_doc.meta["page_number"] = current_page
478
-
479
- # keep the new chunk doc and update the current position
480
541
  new_docs.append(new_doc)
481
- # Advance current_position by chunk length minus overlap.
482
- # split_overlap is in split_units, not chars, so get the actual
483
- # overlap string from _get_overlap() and use its char length.
484
- if self.split_overlap > 0 and split_nr < len(chunks) - 1:
485
- overlap_str, _ = self._get_overlap([doc.content for doc in new_docs]) # type: ignore[misc]
486
- overlap_char_len = len(overlap_str)
487
- else:
488
- overlap_char_len = 0
489
- current_position += len(chunk) - overlap_char_len
490
542
 
491
543
  return new_docs
492
544
 
@@ -498,7 +550,8 @@ class RecursiveDocumentSplitter:
498
550
  :param documents: List of Documents to split.
499
551
  :returns:
500
552
  A dictionary containing a key "documents" with a List of Documents with smaller chunks of text corresponding
501
- to the input documents.
553
+ to the input documents. Each chunk carries a metadata field `page_number` with the page the chunk
554
+ starts on, counting form feed ("\f") characters in the original document.
502
555
  """
503
556
  if not self._is_warmed_up and ("sentence" in self.separators or self.split_units == "token"):
504
557
  self.warm_up()
@@ -253,4 +253,4 @@ class SentenceSplitter:
253
253
  logger.warning("No abbreviations file found for {language}. Using default abbreviations.", language=lang)
254
254
  return []
255
255
 
256
- return abbreviations_file.read_text().split("\n")
256
+ return abbreviations_file.read_text(encoding="utf-8").split("\n")
@@ -89,6 +89,10 @@ class LostInTheMiddleRanker:
89
89
  if not documents:
90
90
  return {"documents": []}
91
91
 
92
+ # Raise an error if any document is not textual
93
+ if any(doc.content is None for doc in documents):
94
+ raise ValueError("Some provided documents are not textual; LostInTheMiddleRanker can process only text.")
95
+
92
96
  top_k = top_k or self.top_k
93
97
  word_count_threshold = word_count_threshold or self.word_count_threshold
94
98
 
@@ -99,10 +103,6 @@ class LostInTheMiddleRanker:
99
103
  if len(documents_to_reorder) == 1:
100
104
  return {"documents": documents_to_reorder}
101
105
 
102
- # Raise an error if any document is not textual
103
- if any(doc.content is None for doc in documents_to_reorder):
104
- raise ValueError("Some provided documents are not textual; LostInTheMiddleRanker can process only text.")
105
-
106
106
  # Initialize word count and indices for the "lost in the middle" order
107
107
  word_count = 0
108
108
  document_index = list(range(len(documents_to_reorder)))
@@ -89,7 +89,7 @@ class Telemetry:
89
89
  CONFIG_PATH.parents[0].mkdir(parents=True, exist_ok=True)
90
90
  self.user_id = str(uuid.uuid4())
91
91
  try:
92
- with open(CONFIG_PATH, "w") as outfile:
92
+ with open(CONFIG_PATH, "w", encoding="utf-8") as outfile:
93
93
  yaml.dump({"user_id": self.user_id}, outfile, default_flow_style=False)
94
94
  except Exception as e:
95
95
  logger.debug(
@@ -1 +0,0 @@
1
- 3.3.0-rc1