haystack-experimental 0.13.0__py3-none-any.whl → 0.14.1__py3-none-any.whl

haystack_experimental/components/agents/human_in_the_loop/user_interfaces.py

@@ -0,0 +1,209 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import json
+from threading import Lock
+from typing import Any, Optional
+
+from haystack.core.serialization import default_to_dict
+from rich.console import Console
+from rich.panel import Panel
+from rich.prompt import Prompt
+
+from haystack_experimental.components.agents.human_in_the_loop.dataclasses import ConfirmationUIResult
+from haystack_experimental.components.agents.human_in_the_loop.types import ConfirmationUI
+
+_ui_lock = Lock()
+
+
+class RichConsoleUI(ConfirmationUI):
+    """Rich console interface for user interaction."""
+
+    def __init__(self, console: Optional[Console] = None):
+        self.console = console or Console()
+
+    def get_user_confirmation(
+        self, tool_name: str, tool_description: str, tool_params: dict[str, Any]
+    ) -> ConfirmationUIResult:
+        """
+        Get user confirmation for tool execution via rich console prompts.
+
+        :param tool_name: The name of the tool to be executed.
+        :param tool_description: The description of the tool.
+        :param tool_params: The parameters to be passed to the tool.
+        :returns: ConfirmationUIResult based on user input.
+        """
+        with _ui_lock:
+            self._display_tool_info(tool_name, tool_description, tool_params)
+            # If wrong input is provided, Prompt.ask will re-prompt
+            choice = Prompt.ask("\nYour choice", choices=["y", "n", "m"], default="y", console=self.console)
+            return self._process_choice(choice, tool_params)
+
+    def _display_tool_info(self, tool_name: str, tool_description: str, tool_params: dict[str, Any]) -> None:
+        """
+        Display tool information and parameters in a rich panel.
+
+        :param tool_name: The name of the tool to be executed.
+        :param tool_description: The description of the tool.
+        :param tool_params: The parameters to be passed to the tool.
+        """
+        lines = [
+            f"[bold yellow]Tool:[/bold yellow] {tool_name}",
+            f"[bold yellow]Description:[/bold yellow] {tool_description}",
+            "\n[bold yellow]Arguments:[/bold yellow]",
+        ]
+
+        if tool_params:
+            for k, v in tool_params.items():
+                lines.append(f"[cyan]{k}:[/cyan] {v}")
+        else:
+            lines.append("  (No arguments)")
+
+        self.console.print(Panel("\n".join(lines), title="🔧 Tool Execution Request", title_align="left"))
+
+    def _process_choice(self, choice: str, tool_params: dict[str, Any]) -> ConfirmationUIResult:
+        """
+        Process the user's choice and return the corresponding ConfirmationUIResult.
+
+        :param choice: The user's choice ('y', 'n', or 'm').
+        :param tool_params: The original tool parameters.
+        :returns:
+            ConfirmationUIResult based on user input.
+        """
+        if choice == "y":
+            return ConfirmationUIResult(action="confirm")
+        elif choice == "m":
+            return self._modify_params(tool_params)
+        else:  # reject
+            feedback = Prompt.ask("Feedback message (optional)", default="", console=self.console)
+            return ConfirmationUIResult(action="reject", feedback=feedback or None)
+
+    def _modify_params(self, tool_params: dict[str, Any]) -> ConfirmationUIResult:
+        """
+        Prompt the user to modify tool parameters.
+
+        :param tool_params: The original tool parameters.
+        :returns:
+            ConfirmationUIResult with modified parameters.
+        """
+        new_params: dict[str, Any] = {}
+        for k, v in tool_params.items():
+            # We don't JSON dump strings to avoid users needing to input extra quotes
+            default_val = json.dumps(v) if not isinstance(v, str) else v
+            while True:
+                new_val = Prompt.ask(f"Modify '{k}'", default=default_val, console=self.console)
+                try:
+                    if isinstance(v, str):
+                        # Always treat input as string
+                        new_params[k] = new_val
+                    else:
+                        # Parse JSON for all non-string types
+                        new_params[k] = json.loads(new_val)
+                    break
+                except json.JSONDecodeError:
+                    self.console.print("[red]❌ Invalid JSON, please try again.[/red]")
+
+        return ConfirmationUIResult(action="modify", new_tool_params=new_params)
+
+    def to_dict(self) -> dict[str, Any]:
+        """
+        Serializes the RichConsoleConfirmationUI to a dictionary.
+
+        :returns:
+            Dictionary with serialized data.
+        """
+        # Note: Console object is not serializable; we store None
+        return default_to_dict(self, console=None)
+
+
+class SimpleConsoleUI(ConfirmationUI):
+    """Simple console interface using standard input/output."""
+
+    def get_user_confirmation(
+        self, tool_name: str, tool_description: str, tool_params: dict[str, Any]
+    ) -> ConfirmationUIResult:
+        """
+        Get user confirmation for tool execution via simple console prompts.
+
+        :param tool_name: The name of the tool to be executed.
+        :param tool_description: The description of the tool.
+        :param tool_params: The parameters to be passed to the tool.
+        """
+        with _ui_lock:
+            self._display_tool_info(tool_name, tool_description, tool_params)
+            valid_choices = {"y", "yes", "n", "no", "m", "modify"}
+            while True:
+                choice = input("Confirm execution? (y=confirm / n=reject / m=modify): ").strip().lower()
+                if choice in valid_choices:
+                    break
+                print("Invalid input. Please enter 'y', 'n', or 'm'.")
+            return self._process_choice(choice, tool_params)
+
+    def _display_tool_info(self, tool_name: str, tool_description: str, tool_params: dict[str, Any]) -> None:
+        """
+        Display tool information and parameters in the console.
+
+        :param tool_name: The name of the tool to be executed.
+        :param tool_description: The description of the tool.
+        :param tool_params: The parameters to be passed to the tool.
+        """
+        print("\n--- Tool Execution Request ---")
+        print(f"Tool: {tool_name}")
+        print(f"Description: {tool_description}")
+        print("Arguments:")
+        if tool_params:
+            for k, v in tool_params.items():
+                print(f"  {k}: {v}")
+        else:
+            print("  (No arguments)")
+        print("-" * 30)
+
+    def _process_choice(self, choice: str, tool_params: dict[str, Any]) -> ConfirmationUIResult:
+        """
+        Process the user's choice and return the corresponding ConfirmationUIResult.
+
+        :param choice: The user's choice ('y', 'n', or 'm').
+        :param tool_params: The original tool parameters.
+        :returns:
+            ConfirmationUIResult based on user input.
+        """
+        if choice in ("y", "yes"):
+            return ConfirmationUIResult(action="confirm")
+        elif choice in ("m", "modify"):
+            return self._modify_params(tool_params)
+        else:  # reject
+            feedback = input("Feedback message (optional): ").strip()
+            return ConfirmationUIResult(action="reject", feedback=feedback or None)
+
+    def _modify_params(self, tool_params: dict[str, Any]) -> ConfirmationUIResult:
+        """
+        Prompt the user to modify tool parameters.
+
+        :param tool_params: The original tool parameters.
+        :returns:
+            ConfirmationUIResult with modified parameters.
+        """
+        new_params: dict[str, Any] = {}
+
+        if not tool_params:
+            print("No parameters to modify, skipping modification.")
+            return ConfirmationUIResult(action="modify", new_tool_params=new_params)
+
+        for k, v in tool_params.items():
+            # We don't JSON dump strings to avoid users needing to input extra quotes
+            default_val = json.dumps(v) if not isinstance(v, str) else v
+            while True:
+                new_val = input(f"Modify '{k}' (current: {default_val}): ").strip() or default_val
+                try:
+                    if isinstance(v, str):
+                        # Always treat input as string
+                        new_params[k] = new_val
+                    else:
+                        # Parse JSON for all non-string types
+                        new_params[k] = json.loads(new_val)
+                    break
+                except json.JSONDecodeError:
+                    print("❌ Invalid JSON, please try again.")
+
+        return ConfirmationUIResult(action="modify", new_tool_params=new_params)

haystack_experimental/components/generators/chat/openai.py

@@ -3,12 +3,12 @@
 # SPDX-License-Identifier: Apache-2.0
 
 from dataclasses import replace
-from typing import Any, Optional, Union
+from typing import Any, Optional
 
 from haystack import component
 from haystack.components.generators.chat.openai import OpenAIChatGenerator as BaseOpenAIChatGenerator
 from haystack.dataclasses import ChatMessage, StreamingCallbackT
-from haystack.tools import Tool, Toolset
+from haystack.tools import ToolsType
 
 from haystack_experimental.utils.hallucination_risk_calculator.dataclasses import HallucinationScoreConfig
 from haystack_experimental.utils.hallucination_risk_calculator.openai_planner import calculate_hallucination_metrics
@@ -59,7 +59,7 @@ class OpenAIChatGenerator(BaseOpenAIChatGenerator):
         streaming_callback: Optional[StreamingCallbackT] = None,
         generation_kwargs: Optional[dict[str, Any]] = None,
         *,
-        tools: Optional[Union[list[Tool], Toolset]] = None,
+        tools: Optional[ToolsType] = None,
         tools_strict: Optional[bool] = None,
         hallucination_score_config: Optional[HallucinationScoreConfig] = None,
     ) -> dict[str, list[ChatMessage]]:
@@ -75,9 +75,8 @@ class OpenAIChatGenerator(BaseOpenAIChatGenerator):
             override the parameters passed during component initialization.
             For details on OpenAI API parameters, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat/create).
         :param tools:
-            A list of tools or a Toolset for which the model can prepare calls. If set, it will override the
-            `tools` parameter set during component initialization. This parameter can accept either a list of
-            `Tool` objects or a `Toolset` instance.
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+            If set, it will override the `tools` parameter provided during initialization.
         :param tools_strict:
             Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
             the schema provided in the `parameters` field of the tool definition, but this may increase latency.
@@ -127,7 +126,7 @@ class OpenAIChatGenerator(BaseOpenAIChatGenerator):
         streaming_callback: Optional[StreamingCallbackT] = None,
         generation_kwargs: Optional[dict[str, Any]] = None,
         *,
-        tools: Optional[Union[list[Tool], Toolset]] = None,
+        tools: Optional[ToolsType] = None,
         tools_strict: Optional[bool] = None,
         hallucination_score_config: Optional[HallucinationScoreConfig] = None,
     ) -> dict[str, list[ChatMessage]]:
@@ -147,9 +146,8 @@ class OpenAIChatGenerator(BaseOpenAIChatGenerator):
             override the parameters passed during component initialization.
             For details on OpenAI API parameters, see [OpenAI documentation](https://platform.openai.com/docs/api-reference/chat/create).
         :param tools:
-            A list of tools or a Toolset for which the model can prepare calls. If set, it will override the
-            `tools` parameter set during component initialization. This parameter can accept either a list of
-            `Tool` objects or a `Toolset` instance.
+            A list of Tool and/or Toolset objects, or a single Toolset for which the model can prepare calls.
+            If set, it will override the `tools` parameter provided during initialization.
         :param tools_strict:
             Whether to enable strict schema adherence for tool calls. If set to `True`, the model will follow exactly
             the schema provided in the `parameters` field of the tool definition, but this may increase latency.

haystack_experimental/components/preprocessors/embedding_based_document_splitter.py

@@ -37,22 +37,31 @@ class EmbeddingBasedDocumentSplitter:
     from haystack.components.embedders import SentenceTransformersDocumentEmbedder
     from haystack_experimental.components.preprocessors import EmbeddingBasedDocumentSplitter
 
+    # Create a document with content that has a clear topic shift
     doc = Document(
         content="This is a first sentence. This is a second sentence. This is a third sentence. "
         "Completely different topic. The same completely different topic."
     )
 
+    # Initialize the embedder to calculate semantic similarities
     embedder = SentenceTransformersDocumentEmbedder()
 
+    # Configure the splitter with parameters that control splitting behavior
     splitter = EmbeddingBasedDocumentSplitter(
         document_embedder=embedder,
-        sentences_per_group=2,
-        percentile=0.95,
-        min_length=50,
-        max_length=1000
+        sentences_per_group=2,      # Group 2 sentences before calculating embeddings
+        percentile=0.95,            # Split when cosine distance exceeds 95th percentile
+        min_length=50,              # Merge splits shorter than 50 characters
+        max_length=1000             # Further split chunks longer than 1000 characters
     )
     splitter.warm_up()
     result = splitter.run(documents=[doc])
+
+    # The result contains a list of Document objects, each representing a semantic chunk
+    # Each split document includes metadata: source_id, split_id, and page_number
+    print(f"Original document split into {len(result['documents'])} chunks")
+    for i, split_doc in enumerate(result['documents']):
+        print(f"Chunk {i}: {split_doc.content[:50]}...")
     ```
     """
 
@@ -78,8 +87,11 @@ class EmbeddingBasedDocumentSplitter:
         :param min_length: Minimum length of splits in characters. Splits below this length will be merged.
         :param max_length: Maximum length of splits in characters. Splits above this length will be recursively split.
         :param language: Language for sentence tokenization.
-        :param use_split_rules: Whether to use additional split rules for sentence tokenization.
-        :param extend_abbreviations: Whether to extend NLTK abbreviations.
+        :param use_split_rules: Whether to use additional split rules for sentence tokenization. Applies additional
+            split rules from SentenceSplitter to the sentence spans.
+        :param extend_abbreviations: If True, the abbreviations used by NLTK's PunktTokenizer are extended by a list
+            of curated abbreviations. Currently supported languages are: en, de.
+            If False, the default abbreviations are used.
         """
         self.document_embedder = document_embedder
         self.sentences_per_group = sentences_per_group

haystack_experimental/components/preprocessors/md_header_level_inferrer.py

@@ -0,0 +1,146 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+import re
+
+from haystack import Document, component, logging
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class MarkdownHeaderLevelInferrer:
+    """
+    Infers and rewrites header levels in Markdown text to normalize hierarchy.
+
+    First header → Always becomes level 1 (#)
+    Subsequent headers → Level increases if no content between headers, stays same if content exists
+    Maximum level → Capped at 6 (######)
+
+    ### Usage example
+    ```python
+    from haystack import Document
+    from haystack_experimental.components.preprocessors import MarkdownHeaderLevelInferrer
+
+    # Create a document with uniform header levels
+    text = "## Title\nSome content\n## Section\nMore content\n## Subsection\nFinal content"
+    doc = Document(content=text)
+
+    # Initialize the inferrer and process the document
+    inferrer = MarkdownHeaderLevelInferrer()
+    result = inferrer.run([doc])
+
+    # The headers are now normalized with proper hierarchy
+    print(result["documents"][0].content)
+    > # Title\nSome content\n## Section\nMore content\n### Subsection\nFinal content
+    ```
+    """
+
+    def __init__(self):
+        """Initializes the MarkdownHeaderLevelInferrer."""
+        # handles headers with optional trailing spaces and empty content
+        self._header_pattern = re.compile(r"(?m)^(#{1,6})\s+(.+?)(?:\s*)$")
+
+    @component.output_types(documents=list[Document])
+    def run(self, documents: list[Document]) -> dict:
+        """
+        Infers and rewrites the header levels in the content for documents that use uniform header levels.
+
+        :param documents: list of Document objects to process.
+
+        :returns:
+            dict: a dictionary with the key 'documents' containing the processed Document objects.
+        """
+        if not documents:
+            logger.warning("No documents provided to process")
+            return {"documents": []}
+
+        logger.debug(f"Inferring and rewriting header levels for {len(documents)} documents")
+        processed_docs = [self._process_document(doc) for doc in documents]
+        return {"documents": processed_docs}
+
+    def _process_document(self, doc: Document) -> Document:
+        """
+        Processes a single document, inferring and rewriting header levels.
+
+        :param doc: Document object to process.
+        :returns:
+            Document object with rewritten header levels.
+        """
+        if doc.content is None:
+            logger.warning(f"Document {getattr(doc, 'id', '')} content is None; skipping header level inference.")
+            return doc
+
+        matches = list(re.finditer(self._header_pattern, doc.content))
+        if not matches:
+            logger.info(f"No headers found in document {doc.id}; skipping header level inference.")
+            return doc
+
+        modified_text = MarkdownHeaderLevelInferrer._rewrite_headers(doc.content, matches)
+        logger.info(f"Rewrote {len(matches)} headers with inferred levels in document{doc.id}.")
+        return MarkdownHeaderLevelInferrer._build_final_document(doc, modified_text)
+
+    @staticmethod
+    def _rewrite_headers(content: str, matches: list[re.Match]) -> str:
+        """
+        Rewrites the headers in the content with inferred levels.
+
+        :param content: Original Markdown content.
+        :param matches: List of regex matches for headers.
+        """
+        modified_text = content
+        offset = 0
+        current_level = 1
+
+        for i, match in enumerate(matches):
+            original_header = match.group(0)
+            header_text = match.group(2).strip()
+
+            # Skip empty headers
+            if not header_text:
+                logger.warning(f"Skipping empty header at position {match.start()}")
+                continue
+
+            has_content = MarkdownHeaderLevelInferrer._has_content_between_headers(content, matches, i)
+            inferred_level = MarkdownHeaderLevelInferrer._infer_level(i, current_level, has_content)
+            current_level = inferred_level
+
+            new_header = f"{'#' * inferred_level} {header_text}"
+            start_pos = match.start() + offset
+            end_pos = match.end() + offset
+            modified_text = modified_text[:start_pos] + new_header + modified_text[end_pos:]
+            offset += len(new_header) - len(original_header)
+
+        return modified_text
+
+    @staticmethod
+    def _has_content_between_headers(content: str, matches: list[re.Match], i: int) -> bool:
+        """Checks if there is content between the previous and current header."""
+        if i == 0:
+            return False
+        prev_end = matches[i - 1].end()
+        current_start = matches[i].start()
+        content_between = content[prev_end:current_start]
+        return bool(content_between.strip())
+
+    @staticmethod
+    def _infer_level(i: int, current_level: int, has_content: bool) -> int:
+        """Infers the header level for the current header."""
+        if i == 0:
+            return 1
+        if has_content:
+            return current_level
+        return min(current_level + 1, 6)
+
+    @staticmethod
+    def _build_final_document(doc: Document, new_content: str) -> Document:
+        """Creates a new Document with updated content, preserving other fields."""
+        return Document(
+            id=getattr(doc, "id", "") or "",
+            content=new_content,
+            blob=getattr(doc, "blob", None),
+            meta=getattr(doc, "meta", {}) or {},
+            score=getattr(doc, "score", None),
+            embedding=getattr(doc, "embedding", None),
+        )

haystack_experimental/components/summarizers/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from haystack_experimental.components.summarizers.llm_summarizer import LLMSummarizer
+
+_all_ = ["Summarizer"]