haystack-ai 3.2.0.dev20260904000310__tar.gz → 3.2.0.dev20260906000331__tar.gz

{haystack_ai-3.2.0.dev20260904000310 → haystack_ai-3.2.0.dev20260906000331}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: haystack-ai
-Version: 3.2.0.dev20260904000310
+Version: 3.2.0.dev20260906000331
 Summary: LLM framework to build customizable, production-ready LLM applications. Connect components (models, vector DBs, file converters) to pipelines or agents that can interact with your data.
 Project-URL: CI: GitHub, https://github.com/deepset-ai/haystack/actions
 Project-URL: Docs: RTD, https://haystack.deepset.ai/overview/intro

haystack_ai-3.2.0.dev20260906000331/VERSION.txt

@@ -0,0 +1 @@
+3.2.0.dev20260906000331

{haystack_ai-3.2.0.dev20260904000310 → haystack_ai-3.2.0.dev20260906000331}/haystack/components/preprocessors/document_preprocessor.py

@@ -35,7 +35,7 @@ class DocumentPreprocessor:
         self,
         *,
         # --- DocumentSplitter arguments ---
-        split_by: Literal["function", "page", "passage", "period", "word", "line", "sentence"] = "word",
+        split_by: Literal["function", "page", "passage", "period", "word", "line", "sentence", "token"] = "word",
         split_length: int = 250,
         split_overlap: int = 0,
         split_threshold: int = 0,
@@ -44,6 +44,7 @@ class DocumentPreprocessor:
         language: Language = "en",
         use_split_rules: bool = True,
         extend_abbreviations: bool = True,
+        tokenizer_encoding: str = "o200k_base",
         # --- DocumentCleaner arguments ---
         remove_empty_lines: bool = True,
         remove_extra_whitespaces: bool = True,
@@ -58,7 +59,8 @@ class DocumentPreprocessor:
         Initialize a DocumentPreProcessor that first splits and then cleans documents.
 
         **Splitter Parameters**:
-        :param split_by: The unit of splitting: "function", "page", "passage", "period", "word", "line", or "sentence".
+        :param split_by: The unit of splitting: "function", "page", "passage", "period", "word", "line",
+            "sentence", or "token".
         :param split_length: The maximum number of units (words, lines, pages, and so on) in each split.
         :param split_overlap: The number of overlapping units between consecutive splits.
         :param split_threshold: The minimum number of units per split. If a split is smaller than this, it's merged
@@ -70,6 +72,8 @@ class DocumentPreprocessor:
         :param use_split_rules: Whether to apply additional splitting heuristics for the sentence splitter.
         :param extend_abbreviations: Whether to extend the sentence splitter with curated abbreviations for certain
             languages.
+        :param tokenizer_encoding: The tiktoken encoding to use when `split_by="token"`. Defaults to
+            `"o200k_base"` (current OpenAI models). Only used when `split_by="token"`.
 
         **Cleaner Parameters**:
         :param remove_empty_lines: If `True`, removes empty lines.
@@ -100,6 +104,7 @@ class DocumentPreprocessor:
         self.language = language
         self.use_split_rules = use_split_rules
         self.extend_abbreviations = extend_abbreviations
+        self.tokenizer_encoding = tokenizer_encoding
 
         # Instantiate sub-components
         splitter = DocumentSplitter(
@@ -112,6 +117,7 @@ class DocumentPreprocessor:
             language=self.language,
             use_split_rules=self.use_split_rules,
             extend_abbreviations=self.extend_abbreviations,
+            tokenizer_encoding=self.tokenizer_encoding,
         )
 
         cleaner = DocumentCleaner(
@@ -180,6 +186,7 @@ class DocumentPreprocessor:
             language=self.language,
             use_split_rules=self.use_split_rules,
             extend_abbreviations=self.extend_abbreviations,
+            tokenizer_encoding=self.tokenizer_encoding,
         )
 
     @classmethod

{haystack_ai-3.2.0.dev20260904000310 → haystack_ai-3.2.0.dev20260906000331}/haystack/components/preprocessors/document_splitter.py

@@ -11,8 +11,12 @@ from more_itertools import windowed
 from haystack import Document, component, logging
 from haystack.components.preprocessors.sentence_tokenizer import Language, SentenceSplitter, nltk_imports
 from haystack.core.serialization import default_from_dict, default_to_dict
+from haystack.lazy_imports import LazyImport
 from haystack.utils import deserialize_callable, serialize_callable
 
+with LazyImport("Run 'pip install tiktoken'") as tiktoken_imports:
+    import tiktoken
+
 logger = logging.getLogger(__name__)
 
 # mapping of split by character, 'function' and 'sentence' don't split by character
@@ -54,7 +58,7 @@ class DocumentSplitter:
 
     def __init__(
         self,
-        split_by: Literal["function", "page", "passage", "period", "word", "line", "sentence"] = "word",
+        split_by: Literal["function", "page", "passage", "period", "word", "line", "sentence", "token"] = "word",
         split_length: int = 200,
         split_overlap: int = 0,
         split_threshold: int = 0,
@@ -65,6 +69,7 @@ class DocumentSplitter:
         extend_abbreviations: bool = True,
         *,
         skip_empty_documents: bool = True,
+        tokenizer_encoding: str = "o200k_base",
     ) -> None:
         """
         Initialize DocumentSplitter.
@@ -76,6 +81,7 @@ class DocumentSplitter:
             - `passage` for splitting by double line breaks ("\\n\\n")
             - `line` for splitting each line ("\\n")
             - `sentence` for splitting by NLTK sentence tokenizer
+            - `token` for splitting by token count using tiktoken (requires `pip install tiktoken`)
 
         :param split_length: The maximum number of units in each split.
         :param split_overlap: The number of overlapping units for each split.
@@ -93,6 +99,8 @@ class DocumentSplitter:
         :param skip_empty_documents: Choose whether to skip documents with empty content. Default is True.
             Set to False when downstream components in the Pipeline (like LLMDocumentContentExtractor) can extract text
             from non-textual documents.
+        :param tokenizer_encoding: The tiktoken encoding to use when `split_by="token"`. Defaults to
+            `"o200k_base"` (current OpenAI models). Only used when `split_by="token"`.
         """
 
         self.split_by = split_by
@@ -105,6 +113,7 @@ class DocumentSplitter:
         self.use_split_rules = use_split_rules
         self.extend_abbreviations = extend_abbreviations
         self.skip_empty_documents = skip_empty_documents
+        self.tokenizer_encoding = tokenizer_encoding
 
         self._init_checks(
             split_by=split_by,
@@ -117,6 +126,9 @@ class DocumentSplitter:
         if self._use_sentence_splitter:
             nltk_imports.check()
             self.sentence_splitter: SentenceSplitter | None = None
+        if split_by == "token":
+            tiktoken_imports.check()
+            self._tiktoken_tokenizer: "tiktoken.Encoding | None" = None
 
     def _init_checks(
         self,
@@ -137,7 +149,7 @@ class DocumentSplitter:
         :param respect_sentence_boundary: Whether to respect sentence boundaries when splitting
         :raises ValueError: If any parameter is invalid
         """
-        valid_split_by = ["function", "page", "passage", "period", "word", "line", "sentence"]
+        valid_split_by = ["function", "page", "passage", "period", "word", "line", "sentence", "token"]
         if split_by not in valid_split_by:
             raise ValueError(f"split_by must be one of {', '.join(valid_split_by)}.")
 
@@ -162,7 +174,7 @@ class DocumentSplitter:
 
     def warm_up(self) -> None:
         """
-        Warm up the DocumentSplitter by loading the sentence tokenizer.
+        Warm up the DocumentSplitter by loading the sentence tokenizer or tiktoken encoder.
         """
         if self._use_sentence_splitter and self.sentence_splitter is None:
             self.sentence_splitter = SentenceSplitter(
@@ -171,6 +183,8 @@ class DocumentSplitter:
                 extend_abbreviations=self.extend_abbreviations,
                 keep_white_spaces=True,
             )
+        if self.split_by == "token" and self._tiktoken_tokenizer is None:
+            self._tiktoken_tokenizer = tiktoken.get_encoding(self.tokenizer_encoding)
 
     @component.output_types(documents=list[Document])
     def run(self, documents: list[Document]) -> dict[str, list[Document]]:
@@ -192,6 +206,8 @@ class DocumentSplitter:
         """
         if self._use_sentence_splitter and self.sentence_splitter is None:
             self.warm_up()
+        if self.split_by == "token" and self._tiktoken_tokenizer is None:
+            self.warm_up()
 
         if not isinstance(documents, list) or (documents and not isinstance(documents[0], Document)):
             raise TypeError("DocumentSplitter expects a List of Documents as input.")
@@ -216,6 +232,9 @@ class DocumentSplitter:
         if self.split_by == "function" and self.splitting_function is not None:
             return self._split_by_function(doc)
 
+        if self.split_by == "token":
+            return self._split_by_token(doc)
+
         return self._split_by_character(doc)
 
     def _split_by_nltk_sentence(self, doc: Document) -> list[Document]:
@@ -243,6 +262,56 @@ class DocumentSplitter:
 
         return split_docs
 
+    def _split_by_token(self, doc: Document) -> list[Document]:
+        """
+        Split a document by token count using tiktoken.
+
+        Encodes the full document text to tokens, slices into chunks of `split_length` tokens
+        with `split_overlap` overlap, then decodes each chunk back to a string.
+        """
+        tokens = self._tiktoken_tokenizer.encode(doc.content)  # type: ignore[union-attr, arg-type]
+        if not tokens:
+            if self.skip_empty_documents:
+                return []
+            metadata = deepcopy(doc.meta)
+            metadata["source_id"] = doc.id
+            return self._create_docs_from_splits(
+                text_splits=[""], splits_pages=[1], splits_start_idxs=[0], meta=metadata
+            )
+
+        step = self.split_length - self.split_overlap
+        full_text, offsets = self._tiktoken_tokenizer.decode_with_offsets(tokens)  # type: ignore[union-attr]
+        offsets.append(len(full_text))
+
+        text_splits: list[str] = []
+        splits_pages: list[int] = []
+        splits_start_idxs: list[int] = []
+        cur_page = 1
+        prev_end = 0
+
+        for i in range(0, len(tokens), step):
+            chunk_token_count = min(self.split_length, len(tokens) - i)
+            chunk_start = offsets[i]
+            chunk_end = offsets[i + chunk_token_count]
+
+            if chunk_token_count < self.split_threshold and len(text_splits) > 0:
+                overlap_start = offsets[min(i + self.split_overlap, len(tokens))]
+                text_splits[-1] += full_text[overlap_start:chunk_end]
+            else:
+                text_splits.append(full_text[chunk_start:chunk_end])
+                splits_pages.append(cur_page)
+                splits_start_idxs.append(chunk_start)
+
+            non_overlap_end = offsets[min(i + step, len(tokens))]
+            cur_page += full_text[prev_end:non_overlap_end].count("\f")
+            prev_end = non_overlap_end
+
+        metadata = deepcopy(doc.meta)
+        metadata["source_id"] = doc.id
+        return self._create_docs_from_splits(
+            text_splits=text_splits, splits_pages=splits_pages, splits_start_idxs=splits_start_idxs, meta=metadata
+        )
+
     def _split_by_character(self, doc: Document) -> list[Document]:
         split_at = _CHARACTER_SPLIT_BY_MAPPING[self.split_by]
         units = doc.content.split(split_at)  # type: ignore[union-attr]
@@ -422,6 +491,7 @@ class DocumentSplitter:
             use_split_rules=self.use_split_rules,
             extend_abbreviations=self.extend_abbreviations,
             skip_empty_documents=self.skip_empty_documents,
+            tokenizer_encoding=self.tokenizer_encoding,
         )
         if self.splitting_function:
             serialized["init_parameters"]["splitting_function"] = serialize_callable(self.splitting_function)

{haystack_ai-3.2.0.dev20260904000310 → haystack_ai-3.2.0.dev20260906000331}/haystack/hooks/compaction/hooks.py

@@ -15,7 +15,7 @@ from haystack.core.serialization import (
 )
 from haystack.dataclasses import ChatMessage
 from haystack.hooks.compaction.types import Compactor
-from haystack.hooks.compaction.utils import _estimated_context_tokens, _last_assistant_index
+from haystack.hooks.compaction.utils import _last_assistant_index
 from haystack.token_counters import ApproximateTokenCounter, TokenCounter
 from haystack.tools import ToolsType
 from haystack.utils.deserialization import deserialize_component_inplace
@@ -24,6 +24,32 @@ from haystack.utils.experimental import _experimental
 logger = logging.getLogger(__name__)
 
 
+def _estimated_context_tokens(
+    messages: list[ChatMessage], context_tokens: int, token_counter: TokenCounter, tools: ToolsType | None = None
+) -> int:
+    """
+    Estimate the size of the whole conversation.
+
+    :param messages: The conversation, oldest to newest.
+    :param context_tokens: The `context_tokens` state key, anchored on the provider's own token counting. It accounts
+        for the conversation through the last assistant message, or for the whole conversation when there is none.
+    :param token_counter: The counter to measure the unaccounted messages with.
+    :param tools: Tools whose schemas are sent alongside the messages. These are counted when provider usage is absent.
+    :returns: The estimated total token count.
+    """
+    # Nothing sent yet, or a generator that reports no usage, so count everything.
+    if context_tokens == 0:
+        return token_counter.count(messages=messages, tools=tools)
+    # `context_tokens` accounts for the conversation through the last assistant message, so only the tool result
+    # messages after it still need estimating. If there is no assistant message, `context_tokens` accounts for the
+    # whole conversation, so nothing more needs counting.
+    last_assistant_index = _last_assistant_index(messages=messages)
+    if last_assistant_index < 0:
+        return context_tokens
+    tool_result_messages = messages[last_assistant_index + 1 :]
+    return context_tokens + token_counter.count(messages=tool_result_messages)
+
+
 @_experimental
 class CompactionHook:
     """
@@ -167,6 +193,8 @@ class CompactionHook:
         :returns: The target token amount the messages should be compacted to and the estimated non-message overhead,
             or None when the conversation is not yet large enough to compact.
         """
+        # Estimate the total context size, including the provider's own count of the conversation through the last
+        # assistant message and the tool schemas.
         estimated = _estimated_context_tokens(
             messages=messages, context_tokens=context_tokens, token_counter=self.token_counter, tools=tools
         )
@@ -237,12 +265,11 @@ class CompactionHook:
         # If the original value was 0, leave it unchanged so later steps keep recounting the full request locally.
         if original_context_tokens != 0:
             # Re-estimate the provider-accounted context through the last assistant message, including overhead. If we
-            # added the trailing tool results, a second registered hook could double count them.
-            state.set(
-                "context_tokens",
-                self.token_counter.count(messages=compacted[: _last_assistant_index(messages=compacted) + 1])
-                + estimated_overhead,
-            )
+            # added the trailing tool results, a second registered hook could double count them. If there is no
+            # assistant message, count everything.
+            last_assistant_index = _last_assistant_index(messages=compacted)
+            accounted = compacted if last_assistant_index < 0 else compacted[: last_assistant_index + 1]
+            state.set("context_tokens", self.token_counter.count(messages=accounted) + estimated_overhead)
         logger.debug(
             "Compacted the Agent's conversation at step {step} from {before} to {after} messages, targeting {target} "
             "tokens.",

{haystack_ai-3.2.0.dev20260904000310 → haystack_ai-3.2.0.dev20260906000331}/haystack/hooks/compaction/utils.py

@@ -3,8 +3,6 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from haystack.dataclasses import ChatMessage, ChatRole
-from haystack.token_counters import TokenCounter
-from haystack.tools import ToolsType
 
 # Meta key marking a message that a compactor produced. Its value records which strategy ran.
 _COMPACTION_META_KEY = "context_compaction"
@@ -131,23 +129,3 @@ def _historical_turn_groups(messages: list[ChatMessage], system_end: int, task_i
         list(range(start, end))
         for start, end in _historical_turn_spans(messages=messages, start=system_end, end=historical_end)
     ]
-
-
-def _estimated_context_tokens(
-    messages: list[ChatMessage], context_tokens: int, token_counter: TokenCounter, tools: ToolsType | None = None
-) -> int:
-    """
-    Estimate the size of the whole conversation.
-
-    :param messages: The conversation, oldest to newest.
-    :param context_tokens: The `context_tokens` state key which is computed using the provider's own token counting.
-    :param token_counter: The counter to measure the unaccounted messages with.
-    :param tools: Tools whose schemas are sent alongside the messages. These are counted when provider usage is absent.
-    :returns: The estimated total token count.
-    """
-    # Nothing sent yet, or a generator that reports no usage, so count everything.
-    if context_tokens == 0:
-        return token_counter.count(messages=messages, tools=tools)
-    # Only need to estimate the tool result messages after the last assistant message
-    tool_result_messages = messages[_last_assistant_index(messages=messages) + 1 :]
-    return context_tokens + token_counter.count(messages=tool_result_messages)

{haystack_ai-3.2.0.dev20260904000310 → haystack_ai-3.2.0.dev20260906000331}/haystack/hooks/tool_result_offloading/hooks.py

@@ -243,8 +243,8 @@ class ToolResultOffloadHook:
 
         A message is left as-is when it is not a tool result, when the result is an error (including `before_tool`
         human-in-the-loop rejections), when it was already offloaded (e.g. another offload hook under `after_tool`
-        handled it), when no policy applies, when the result is empty, when the result carries image or file content
-        that `store` cannot store, or when the policy declines to offload.
+        handled it), when no policy applies, when the result is empty (no content, or nothing but empty text), when
+        the result carries image or file content that `store` cannot store, or when the policy declines to offload.
 
         Otherwise the result is written to `store` and the message is rebuilt with a pointer in place of the full
         result, preserving its origin and error flag and marking it offloaded. Each part of the result goes to its own
@@ -273,7 +273,9 @@ class ToolResultOffloadHook:
         content_blocks: list[TextContent | ImageContent | FileContent] = (
             [TextContent(text=result.result)] if isinstance(result.result, str) else list(result.result)
         )
-        if not content_blocks:
+        # A result made up of nothing but empty text has nothing worth storing, so it stays in context. `all` also
+        # covers a result with no content blocks at all.
+        if all(isinstance(content_block, TextContent) and not content_block.text for content_block in content_blocks):
             return message
 
         # Check whether the store can store binary content before offloading an image or file result. A text-only store

haystack_ai-3.2.0.dev20260904000310/VERSION.txt

@@ -1 +0,0 @@
-3.2.0.dev20260904000310