haystack-ai 3.3.0rc1__tar.gz → 3.4.0__tar.gz

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/PKG-INFO

@@ -1,6 +1,6 @@
 Metadata-Version: 2.5
 Name: haystack-ai
-Version: 3.3.0-rc1
+Version: 3.4.0
 Summary: LLM framework to build customizable, production-ready LLM applications. Connect components (models, vector DBs, file converters) to pipelines or agents that can interact with your data.
 Project-URL: CI: GitHub, https://github.com/deepset-ai/haystack/actions
 Project-URL: Docs: RTD, https://haystack.deepset.ai/overview/intro
@@ -18,13 +18,13 @@ Classifier: License :: OSI Approved :: Apache Software License
 Classifier: Operating System :: OS Independent
 Classifier: Programming Language :: Python
 Classifier: Programming Language :: Python :: 3
-Classifier: Programming Language :: Python :: 3.10
 Classifier: Programming Language :: Python :: 3.11
 Classifier: Programming Language :: Python :: 3.12
 Classifier: Programming Language :: Python :: 3.13
 Classifier: Programming Language :: Python :: 3.14
+Classifier: Programming Language :: Python :: 3.15
 Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
-Requires-Python: >=3.10
+Requires-Python: >=3.11
 Requires-Dist: anyio>=4.14.2
 Requires-Dist: docstring-parser
 Requires-Dist: filetype
@@ -43,7 +43,6 @@ Requires-Dist: python-dateutil
 Requires-Dist: pyyaml
 Requires-Dist: tenacity!=8.4.0
 Requires-Dist: tqdm
-Requires-Dist: typing-extensions>=4.7
 Description-Content-Type: text/markdown
 
 <div align="center">

haystack_ai-3.4.0/VERSION.txt

@@ -0,0 +1 @@
+3.4.0

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/haystack/components/agents/agent.py

@@ -3,6 +3,7 @@
 # SPDX-License-Identifier: Apache-2.0
 
 import inspect
+from collections import Counter
 from dataclasses import dataclass
 from typing import Any, Literal, cast
 
@@ -28,7 +29,12 @@ from haystack.components.agents.utils import (
 )
 from haystack.components.builders import ChatPromptBuilder
 from haystack.components.generators.chat.types import ChatGenerator
-from haystack.core.serialization import component_to_dict, default_from_dict, default_to_dict
+from haystack.core.serialization import (
+    component_to_dict,
+    default_from_dict,
+    default_to_dict,
+    generate_qualified_class_name,
+)
 from haystack.dataclasses import ChatMessage, ChatRole, StreamingCallbackT, select_streaming_callback
 from haystack.hooks.invocation import _run_hooks, _run_hooks_async
 from haystack.hooks.protocol import (
@@ -51,13 +57,18 @@ from haystack.hooks.utils import (
     warm_up_hooks_async,
 )
 from haystack.tools import (
+    ComponentTool,
+    Tool,
     Toolset,
     ToolsType,
     _check_duplicate_tool_names,
+    close_tools,
+    close_tools_async,
     deserialize_tools_or_toolset_inplace,
     flatten_tools_or_toolsets,
     serialize_tools_or_toolset,
     warm_up_tools,
+    warm_up_tools_async,
 )
 from haystack.utils.async_utils import _execute_component_async
 from haystack.utils.callable_serialization import deserialize_callable, serialize_callable
@@ -199,6 +210,34 @@ def _pending_tool_call_messages_from_state(state: State) -> list[ChatMessage]:
     return [last_message] if last_message.tool_calls else []
 
 
+def _get_tool_telemetry_data(tool: Tool) -> dict[str, Any]:
+    """
+    Describe a single tool for telemetry.
+
+    Descriptions are deliberately left out: they are free text that can hold personal or confidential data.
+    """
+    data = {"type": generate_qualified_class_name(type(tool)), "name": tool.name}
+    if isinstance(tool, ComponentTool):
+        data["component"] = generate_qualified_class_name(type(tool._component))
+    return data
+
+
+def _get_tools_telemetry_data(tools: ToolsType) -> dict[str, Any]:
+    """
+    Summarize the tools for telemetry.
+
+    Call it only after the tools are warmed up: flattening iterates the Toolsets, and some of them (e.g.
+    SearchableToolset) warm up on iteration, while others (e.g. MCPToolset) only load their tools in `warm_up()`.
+    """
+    toolsets = [tools] if isinstance(tools, Toolset) else [entry for entry in tools if isinstance(entry, Toolset)]
+    flat_tools = flatten_tools_or_toolsets(tools=tools)
+    return {
+        "count": len(flat_tools),
+        "tools": [_get_tool_telemetry_data(tool) for tool in flat_tools],
+        "toolset_types": dict(Counter(generate_qualified_class_name(type(toolset)) for toolset in toolsets)),
+    }
+
+
 @dataclass(kw_only=True)
 class _ExecutionContext:
     """
@@ -241,7 +280,7 @@ class Agent:
 
     ```python
     from haystack.components.agents import Agent
-    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.generators.chat import OpenAIResponsesChatGenerator
     from haystack.components.generators.utils import print_streaming_chunk
     from haystack.dataclasses import ChatMessage
     from haystack.tools import tool
@@ -272,7 +311,7 @@ class Agent:
             "You are a helpful assistant. Use the 'search' tool to find information "
             "about a user's question and the 'calculator' tool to perform math."
         ),
-        chat_generator=OpenAIChatGenerator(),
+        chat_generator=OpenAIResponsesChatGenerator(),
         tools=[search, calculator],
         streaming_callback=print_streaming_chunk,
     )
@@ -293,7 +332,7 @@ class Agent:
 
     ```python
     from haystack.components.agents import Agent
-    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.generators.chat import OpenAIResponsesChatGenerator
     from haystack.tools import tool
     from typing import Annotated
 
@@ -308,7 +347,7 @@ class Agent:
         return f"[Translated '{text}' to {target_language}]"
 
     agent = Agent(
-        chat_generator=OpenAIChatGenerator(),
+        chat_generator=OpenAIResponsesChatGenerator(),
         tools=[translate],
         system_prompt="You are a helpful translation assistant.",
         user_prompt=\"\"\"{% message role="user"%}
@@ -341,13 +380,13 @@ class Agent:
     - `on_exit`: runs when the Agent is about to stop on an exit condition. An `on_exit` hook can keep the Agent
       running by setting `state.set("continue_run", True)`.
 
-    Use the `@hook` decorator to build a hook from a function. This `on_exit` hook keeps the Agent running until a
-    required tool has been called.
+    Use the `@hook` decorator to build a hook from a function. This `on_exit` hook keeps the Agent running when it
+    stops with a text reply, until it finishes by calling `save_result`.
 
     ```python
     from haystack.components.agents import Agent
     from haystack.components.agents.state import State
-    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.generators.chat import OpenAIResponsesChatGenerator
     from haystack.dataclasses import ChatMessage
     from haystack.hooks import hook
     from haystack.tools import tool
@@ -363,14 +402,15 @@ class Agent:
 
     @hook
     def require_save(state: State) -> None:
-        if state.get("tool_call_counts", {}).get("save_result", 0) == 0:
-            state.set("messages", [ChatMessage.from_system("Call `save_result` before finishing.")])
+        if state.data.get("exit_reason") == "text":
+            state.set("messages", [ChatMessage.from_user("Call `save_result` before finishing.")])
             state.set("continue_run", True)  # keep the Agent running instead of stopping
 
 
     agent = Agent(
-        chat_generator=OpenAIChatGenerator(),
+        chat_generator=OpenAIResponsesChatGenerator(),
         tools=[save_result],
+        exit_conditions=["save_result"],
         hooks={"on_exit": [require_save]},
     )
     ```
@@ -601,7 +641,7 @@ class Agent:
 
     async def warm_up_async(self) -> None:
         """Warm up the tools, hooks, and the underlying chat generator on the serving event loop."""
-        warm_up_tools(tools=self.tools)
+        await warm_up_tools_async(tools=self.tools)
         await warm_up_hooks_async(self.hooks)
         if hasattr(self.chat_generator, "warm_up_async"):
             await self.chat_generator.warm_up_async()
@@ -609,13 +649,15 @@ class Agent:
             self.chat_generator.warm_up()
 
     def close(self) -> None:
-        """Release the hooks' and the underlying chat generator's resources."""
+        """Release tools, hooks, and chat generator resources."""
+        close_tools(tools=self.tools)
         close_hooks(self.hooks)
         if hasattr(self.chat_generator, "close"):
             self.chat_generator.close()
 
     async def close_async(self) -> None:
-        """Release the hooks' and the underlying chat generator's async resources."""
+        """Release async tools, hooks, and chat generator resources."""
+        await close_tools_async(tools=self.tools)
         await close_hooks_async(self.hooks)
         if hasattr(self.chat_generator, "close_async"):
             await self.chat_generator.close_async()
@@ -633,6 +675,15 @@ class Agent:
         params: dict[str, Any] = {name: getattr(self, name) for name in init_params if name != "self"}
         return type(self)(**{**params, **overrides})
 
+    def _get_telemetry_data(self) -> dict[str, Any]:
+        """
+        Data that is sent to Posthog for usage analytics.
+        """
+        chat_generator_data: dict[str, Any] = {"type": generate_qualified_class_name(type(self.chat_generator))}
+        if hasattr(self.chat_generator, "_get_telemetry_data"):
+            chat_generator_data.update(self.chat_generator._get_telemetry_data())
+        return {"chat_generator": chat_generator_data, "tools": _get_tools_telemetry_data(self.tools)}
+
     def to_dict(self) -> dict[str, Any]:
         """
         Serialize the component to a dictionary.
@@ -817,9 +868,6 @@ class Agent:
 
         if isinstance(tools, (Toolset, list)):
             selected = cast(ToolsType, tools)  # mypy can't narrow the Union type from the isinstance checks
-            # Per-run tools are not covered by the Agent's own warm_up(), so warm them up here.
-            # warm_up() is expected to be idempotent, so re-warming on every run is cheap.
-            warm_up_tools(tools=selected)
             return _spawn_tools(tools=selected)
 
         raise TypeError(
@@ -847,6 +895,8 @@ class Agent:
             precedence, keys set only at initialization are kept.
         :param tools: Optional list of Tool objects, a Toolset, or list of tool names to use for this run.
             When passing tool names, tools are selected from the Agent's originally configured tools.
+            Tool and Toolset objects passed here are warmed up automatically; the caller is responsible for
+            closing them if they hold resources.
         :param hook_context: Optional dictionary of request-scoped resources made available to hooks via
             `state.data.get("hook_context")`. Useful in web/server environments to provide per-request objects
             (e.g., WebSocket connections, async queues, Redis pub/sub clients) that a hook can use, for
@@ -874,6 +924,12 @@ class Agent:
         """
         agent_inputs = {"messages": messages, "streaming_callback": streaming_callback, **kwargs}
         self.warm_up()
+        # warm up tools passed at runtime
+        if isinstance(tools, Toolset):
+            tools_to_warm_up: ToolsType = tools
+        else:
+            tools_to_warm_up = [tool for tool in tools or [] if not isinstance(tool, str)]
+        warm_up_tools(tools=tools_to_warm_up)
 
         exe_context = self._initialize_fresh_execution(
             messages=messages,
@@ -934,6 +990,9 @@ class Agent:
             with the `generation_kwargs` passed at the chat generator's initialization: keys provided here take
             precedence, keys set only at initialization are kept.
         :param tools: Optional list of Tool objects, a Toolset, or list of tool names to use for this run.
+            When passing tool names, tools are selected from the Agent's originally configured tools.
+            Tool and Toolset objects passed here are warmed up automatically; the caller is responsible for
+            closing them if they hold resources.
         :param hook_context: Optional dictionary of request-scoped resources made available to hooks via
             `state.data.get("hook_context")`. Useful in web/server environments to provide per-request objects
             (e.g., WebSocket connections, async queues, Redis pub/sub clients) that a hook can use, for
@@ -961,6 +1020,12 @@ class Agent:
         """
         agent_inputs = {"messages": messages, "streaming_callback": streaming_callback, **kwargs}
         await self.warm_up_async()
+        # warm up tools passed at runtime
+        if isinstance(tools, Toolset):
+            tools_to_warm_up: ToolsType = tools
+        else:
+            tools_to_warm_up = [tool for tool in tools or [] if not isinstance(tool, str)]
+        await warm_up_tools_async(tools=tools_to_warm_up)
 
         exe_context = self._initialize_fresh_execution(
             messages=messages,

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/haystack/components/agents/tool_calling.py

@@ -17,7 +17,7 @@ from haystack.dataclasses import ChatMessage, ToolCall
 from haystack.dataclasses.streaming_chunk import StreamingCallbackT, StreamingChunk, _invoke_streaming_callback
 from haystack.tools import ComponentTool, Tool, ToolsType, _check_duplicate_tool_names, flatten_tools_or_toolsets
 from haystack.tools.errors import ToolInvocationError
-from haystack.tools.parameters_schema_utils import _unwrap_optional
+from haystack.tools.parameters_schema_utils import _is_state_annotation
 from haystack.tracing.utils import _serializable_value
 
 logger = logging.getLogger(__name__)
@@ -187,7 +187,7 @@ def _create_tool_span(tool: Tool, tool_call: ToolCall, parent_span: tracing.Span
     "execute tool" span per invocation
     (https://github.com/open-telemetry/semantic-conventions-genai/blob/main/docs/gen-ai/gen-ai-agent-spans.md#execute-tool-span),
     and tracing backends such as Langfuse follow the same model. The span is tagged with the tool's identity; the caller
-    adds the call arguments and result as content tags.
+    adds the call arguments and result as content tags, and sets `haystack.tool.error` to `True` when the call fails.
 
     The parent is passed in rather than read from the tracer here: the calls of a step run concurrently, so a tracer
     that tracks the active span globally would report a sibling tool span as current and the spans would end up chained
@@ -218,6 +218,7 @@ def _make_context_bound_invoke(
             try:
                 result = tool.invoke(**args)
             except ToolInvocationError as e:
+                span.set_tag("haystack.tool.error", True)
                 span.set_content_tag("haystack.agent.step.tool.output", {"error": str(e)})
                 return e
             span.set_content_tag("haystack.agent.step.tool.output", result)
@@ -255,6 +256,7 @@ def _make_bounded_invoke_async(
                 try:
                     result = await tool.invoke_async(**args)
                 except ToolInvocationError as e:
+                    span.set_tag("haystack.tool.error", True)
                     span.set_content_tag("haystack.agent.step.tool.output", {"error": str(e)})
                     return e
                 span.set_content_tag("haystack.agent.step.tool.output", result)
@@ -307,7 +309,7 @@ def _inject_state_args(tool: Tool, llm_args: dict[str, Any], state: State) -> di
 
     # We also inject the full State object for any parameter annotated as State
     for param_name, param_type in func_params.items():
-        if _unwrap_optional(param_type) is State:
+        if _is_state_annotation(param_type):
             final_args[param_name] = state
 
     return final_args
@@ -424,7 +426,7 @@ def _state_io_for_call(tool: Tool, llm_args: dict[str, Any]) -> tuple[_StateKeys
     """
     func_params = _get_func_params(tool)
     # Check if State is in func_params
-    if any(_unwrap_optional(param_type) is State for param_type in func_params.values()):
+    if any(_is_state_annotation(param_type) for param_type in func_params.values()):
         return _ALL_STATE_KEYS, _ALL_STATE_KEYS
 
     # Calculate reads

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/haystack/components/agents/utils.py

@@ -115,7 +115,7 @@ def _select_tools_by_name(configured_tools: ToolsType, names: list[str]) -> list
     Standalone Tools are kept when their name is requested. A Toolset with run-scoped state (one overriding
     `spawn()`, such as SearchableToolset) is replaced by a per-run copy carrying the requested names, so its
     dynamic behavior (search/lazy-loading) is preserved without mutating the shared, configured Toolset. Any
-    other Toolset is warmed up and reduced to the matching Tools.
+    other Toolset is reduced to the matching Tools. The caller must warm up the tools before selection.
 
     :param configured_tools: The tools configured on the Agent.
     :param names: The requested tool names.

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/haystack/components/builders/chat_prompt_builder.py

@@ -68,13 +68,13 @@ class ChatPromptBuilder:
 
     ```python
     from haystack.components.builders import ChatPromptBuilder
-    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.generators.chat import OpenAIResponsesChatGenerator
     from haystack.dataclasses import ChatMessage
     from haystack import Pipeline
 
     # no parameter init, we don't use any runtime template variables
     prompt_builder = ChatPromptBuilder()
-    llm = OpenAIChatGenerator(model="gpt-5-mini")
+    llm = OpenAIResponsesChatGenerator(model="gpt-5.6-luna")
 
     pipe = Pipeline()
     pipe.add_component("prompt_builder", prompt_builder)
@@ -88,26 +88,18 @@ class ChatPromptBuilder:
 
     res = pipe.run(data={"prompt_builder": {"template_variables": {"location": location, "language": language},
                                         "template": messages}})
-    print(res)
-    # >> {'llm': {'replies': [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text=
-    # "Berlin is the capital city of Germany and one of the most vibrant
-    # and diverse cities in Europe. Here are some key things to know...Enjoy your time exploring the vibrant and dynamic
-    # capital of Germany!")], _name=None, _meta={'model': 'gpt-5-mini',
-    # 'index': 0, 'finish_reason': 'stop', 'usage': {'prompt_tokens': 27, 'completion_tokens': 681, 'total_tokens':
-    # 708}})]}}
+    print(res["llm"]["replies"][0].text)
+    # >> Berlin is the capital city of Germany and one of the most vibrant and diverse cities in Europe. Here are some
+    # >> key things to know...Enjoy your time exploring the vibrant and dynamic capital of Germany!
 
     messages = [system_message, ChatMessage.from_user("What's the weather forecast for {{location}} in the next {{day_count}} days?")]
 
     res = pipe.run(data={"prompt_builder": {"template_variables": {"location": location, "day_count": "5"},
                                         "template": messages}})
 
-    print(res)
-    # >> {'llm': {'replies': [ChatMessage(_role=<ChatRole.ASSISTANT: 'assistant'>, _content=[TextContent(text=
-    # "Here is the weather forecast for Berlin in the next 5
-    # days:\\n\\nDay 1: Mostly cloudy with a high of 22°C (72°F) and...so it's always a good idea to check for updates
-    # closer to your visit.")], _name=None, _meta={'model': 'gpt-5-mini',
-    # 'index': 0, 'finish_reason': 'stop', 'usage': {'prompt_tokens': 37, 'completion_tokens': 201,
-    # 'total_tokens': 238}})]}}
+    print(res["llm"]["replies"][0].text)
+    # >> Here is the weather forecast for Berlin in the next 5 days: Day 1: Mostly cloudy with a high of 22°C (72°F)
+    # >> and...so it's always a good idea to check for updates closer to your visit.
     ```
 
     #### String prompt template

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/haystack/components/builders/prompt_builder.py

@@ -47,7 +47,7 @@ class PromptBuilder:
     ```python
     from haystack import Pipeline, Document
     from haystack.utils import Secret
-    from haystack.components.generators.chat import OpenAIChatGenerator
+    from haystack.components.generators.chat import OpenAIResponsesChatGenerator
     from haystack.components.builders.prompt_builder import PromptBuilder
 
     # in a real world use case documents could come from a retriever, web, or any other source
@@ -64,7 +64,7 @@ class PromptBuilder:
         \"\"\"
     p = Pipeline()
     p.add_component(instance=PromptBuilder(template=prompt_template), name="prompt_builder")
-    p.add_component(instance=OpenAIChatGenerator(api_key=Secret.from_env_var("OPENAI_API_KEY")), name="llm")
+    p.add_component(instance=OpenAIResponsesChatGenerator(api_key=Secret.from_env_var("OPENAI_API_KEY")), name="llm")
     p.connect("prompt_builder", "llm")
 
     question = "Where does Joe live?"

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/haystack/components/converters/csv.py

@@ -215,7 +215,7 @@ class CSVToDocument:
             Remaining row columns are added to ``meta`` with collision-safe
             keys (prefixed with ``csv_`` if needed).
         """
-        row_meta = dict(base_meta)
+        row_meta = {**base_meta, "row_number": row_index}
 
         # content (strict: content_column must exist; validated by caller)
         content = self._safe_value(row.get(content_column))
@@ -235,5 +235,4 @@ class CSVToDocument:
                     suffix += 1
             row_meta[key_to_use] = self._safe_value(v)
 
-        row_meta["row_number"] = row_index
         return Document(content=content, meta=row_meta)

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/haystack/components/converters/docx.py

@@ -19,6 +19,14 @@ from haystack.lazy_imports import LazyImport
 
 logger = logging.getLogger(__name__)
 
+_WORD_NS = "http://schemas.openxmlformats.org/wordprocessingml/2006/main"
+_MARKUP_COMPATIBILITY_NS = "http://schemas.openxmlformats.org/markup-compatibility/2006"
+# A text box keeps its paragraphs here, nested under the drawing inside a run.
+_TEXT_BOX_CONTENT_TAG = f"{{{_WORD_NS}}}txbxContent"
+# Word writes a text box twice inside `mc:AlternateContent`: a `wps:txbx` under
+# `mc:Choice` and the same text as a VML text box under `mc:Fallback`.
+_FALLBACK_TAG = f"{{{_MARKUP_COMPATIBILITY_NS}}}Fallback"
+
 # A Markdown table row ends at a line break and its columns are separated by pipes, so
 # neither can survive inside a cell.
 _MARKDOWN_CELL_BREAK_PATTERN = re.compile(r"\s*(?:\r\n|\r|\n)\s*")
@@ -251,6 +259,7 @@ class DOCXToDocument:
                 else:
                     para_text = self._process_links_in_paragraph(paragraph)
                 elements.append(para_text)
+                elements.extend(self._extract_text_boxes(element, document))
             elif element.tag.endswith("tbl"):
                 table = docx.table.Table(element, document)
                 table_str = (
@@ -262,6 +271,34 @@ class DOCXToDocument:
 
         return elements
 
+    def _extract_text_boxes(self, element: Any, document: "DocxDocument") -> list[str]:
+        """
+        Extracts the text of any text box anchored to a paragraph.
+
+        A text box holds its own paragraphs and tables under `w:txbxContent`, nested in
+        the drawing inside a run, so the anchoring paragraph's text never reaches them.
+
+        :param element: The `w:p` element to look under.
+        :param document: The DOCX Document object.
+        :returns: List of strings, one per paragraph or table found inside a text box.
+        """
+        elements = []
+        for text_box in element.iter(_TEXT_BOX_CONTENT_TAG):
+            # Skip the `mc:Fallback` copy, or the same text is emitted twice.
+            if any(ancestor.tag == _FALLBACK_TAG for ancestor in text_box.iterancestors()):
+                continue
+            for child in text_box:
+                if child.tag.endswith("p"):
+                    elements.append(self._process_links_in_paragraph(Paragraph(child, document)))
+                elif child.tag.endswith("tbl"):
+                    table = docx.table.Table(child, document)
+                    elements.append(
+                        self._table_to_markdown(table)
+                        if self.table_format == DOCXTableFormat.MARKDOWN
+                        else self._table_to_csv(table)
+                    )
+        return elements
+
     def _process_paragraph_with_page_breaks(self, paragraph: "Paragraph") -> str:
         """
         Processes a paragraph with page breaks.
@@ -303,10 +340,12 @@ class DOCXToDocument:
             if isinstance(content, Run):
                 text += content.text
             elif isinstance(content, Hyperlink):
+                # Use `.url` instead of `.address` so a URL fragment stored
+                # separately by Word (in `w:anchor`) is not dropped.
                 if self.link_format == DOCXLinkFormat.MARKDOWN:
-                    formatted_link = f"[{content.text}]({content.address})"
+                    formatted_link = f"[{content.text}]({content.url})"
                 else:  # PLAIN format
-                    formatted_link = f"{content.text} ({content.address})"
+                    formatted_link = f"{content.text} ({content.url})"
                 text += formatted_link
 
         return text

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/haystack/components/converters/html.py

@@ -36,7 +36,10 @@ class HTMLToDocument:
     """
 
     def __init__(
-        self, extraction_kwargs: dict[str, Any] | None = None, store_full_path: bool = False, encoding: str = "utf-8"
+        self,
+        extraction_kwargs: dict[str, Any] | None = None,
+        store_full_path: bool = False,
+        encoding: str | None = None,
     ) -> None:
         """
         Create an HTMLToDocument component.
@@ -48,8 +51,9 @@ class HTMLToDocument:
         If True, the full path of the file is stored in the metadata of the document.
         If False, only the file name is stored.
         :param encoding:
-            The default encoding to use when converting HTML files. If the encoding is specified in the metadata of a
-            source ByteStream, it overrides this value.
+            The encoding to use when decoding HTML files. If None (the default), Trafilatura detects the encoding.
+            If detection produces incorrect characters, install `trafilatura[all]` for additional encoding detection
+            support, or set this parameter if you know the encoding.
         """
         trafilatura_import.check()
 
@@ -124,8 +128,9 @@ class HTMLToDocument:
                 continue
 
             try:
-                encoding = bytestream.meta.get("encoding", self.encoding)
-                text = extract(bytestream.data.decode(encoding), **merged_extraction_kwargs)
+                html_bytes = bytestream.data
+                html_content = html_bytes.decode(self.encoding) if self.encoding is not None else html_bytes
+                text = extract(html_content, **merged_extraction_kwargs)
             except Exception as conversion_e:
                 logger.warning(
                     "Failed to extract text from {source}. Skipping it. Error: {error}",
@@ -134,6 +139,10 @@ class HTMLToDocument:
                 )
                 continue
 
+            if text is None:
+                logger.warning("No text could be extracted from {source}. Skipping it.", source=source)
+                continue
+
             merged_metadata = {**bytestream.meta, **metadata}
 
             if not self.store_full_path and "file_path" in bytestream.meta:

{haystack_ai-3.3.0rc1 → haystack_ai-3.4.0}/haystack/components/converters/image/image_utils.py

@@ -7,9 +7,7 @@ import mimetypes
 from collections import defaultdict
 from io import BytesIO
 from pathlib import Path
-from typing import TypedDict, Union
-
-from typing_extensions import NotRequired
+from typing import NotRequired, TypedDict, Union
 
 from haystack import logging
 from haystack.dataclasses import ByteStream, Document
@@ -162,54 +160,55 @@ def _convert_pdf_to_images(
 
     resolved_page_range = page_range or range(1, num_pages + 1)
 
-    for page_number in resolved_page_range:
-        if page_number < 1 or page_number > num_pages:
-            logger.warning(
-                "Page {page_number} is out of range for the PDF file {file_path}. Skipping it.",
-                page_number=page_number,
-                file_path=bytestream.meta.get("file_path"),
-            )
-            continue
-
-        # Get dimensions of the page
-        page = pdf[max(page_number - 1, 0)]  # Adjust for 0-based indexing
-        _, _, width, height = page.get_mediabox()
+    try:
+        for page_number in resolved_page_range:
+            if page_number < 1 or page_number > num_pages:
+                logger.warning(
+                    "Page {page_number} is out of range for the PDF file {file_path}. Skipping it.",
+                    page_number=page_number,
+                    file_path=bytestream.meta.get("file_path"),
+                )
+                continue
 
-        target_resolution_dpi = 300.0
+            # Get dimensions of the page
+            page = pdf[max(page_number - 1, 0)]  # Adjust for 0-based indexing
+            _, _, width, height = page.get_mediabox()
 
-        # From pypdfium2 docs: scale (float) – A factor scaling the number of pixels per PDF canvas unit. This defines
-        # the resolution of the image. To convert a DPI value to a scale factor, multiply it by the size of 1 canvas
-        # unit in inches (usually 1/72in).
-        # https://pypdfium2.readthedocs.io/en/stable/python_api.html#pypdfium2._helpers.page.PdfPage.render
-        target_scale = target_resolution_dpi / 72.0
+            target_resolution_dpi = 300.0
 
-        # Calculate potential pixels for target_dpi
-        pixels_for_target_scale = width * height * target_scale**2
+            # From pypdfium2 docs: scale (float) – A factor scaling the number of pixels per PDF canvas unit. This
+            # defines the resolution of the image. To convert a DPI value to a scale factor, multiply it by the size of
+            # 1 canvas unit in inches (usually 1/72in).
+            # https://pypdfium2.readthedocs.io/en/stable/python_api.html#pypdfium2._helpers.page.PdfPage.render
+            target_scale = target_resolution_dpi / 72.0
 
-        pil_max_pixels = PILImage.MAX_IMAGE_PIXELS or int(1024 * 1024 * 1024 // 4 // 3)
-        # 90% of PIL's default limit to prevent borderline cases
-        pixel_limit = pil_max_pixels * 0.9
+            # Calculate potential pixels for target_dpi
+            pixels_for_target_scale = width * height * target_scale**2
 
-        scale = target_scale
-        if pixels_for_target_scale > pixel_limit:
-            logger.info(
-                "Large PDF detected ({pixels:.2f} pixels). Resizing the image to fit the pixel limit.",
-                pixels=pixels_for_target_scale,
-            )
-            scale = (pixel_limit / (width * height)) ** 0.5
+            pil_max_pixels = PILImage.MAX_IMAGE_PIXELS or int(1024 * 1024 * 1024 // 4 // 3)
+            # 90% of PIL's default limit to prevent borderline cases
+            pixel_limit = pil_max_pixels * 0.9
 
-        pdf_bitmap = page.render(scale=scale)
+            scale = target_scale
+            if pixels_for_target_scale > pixel_limit:
+                logger.info(
+                    "Large PDF detected ({pixels:.2f} pixels). Resizing the image to fit the pixel limit.",
+                    pixels=pixels_for_target_scale,
+                )
+                scale = (pixel_limit / (width * height)) ** 0.5
 
-        image: "Image" = pdf_bitmap.to_pil()
-        pdf_bitmap.close()
-        if size is not None:
-            # Set reducing_gap=None to disable multi-step shrink; better quality.
-            # https://pillow.readthedocs.io/en/latest/reference/Image.html#PIL.Image.Image.thumbnail
-            image.thumbnail(size=size, reducing_gap=None)
+            pdf_bitmap = page.render(scale=scale)
 
-        all_pdf_images.append((page_number, image))
+            image: "Image" = pdf_bitmap.to_pil()
+            pdf_bitmap.close()
+            if size is not None:
+                # Set reducing_gap=None to disable multi-step shrink; better quality.
+                # https://pillow.readthedocs.io/en/latest/reference/Image.html#PIL.Image.Image.thumbnail
+                image.thumbnail(size=size, reducing_gap=None)
 
-    pdf.close()
+            all_pdf_images.append((page_number, image))
+    finally:
+        pdf.close()
 
     if return_base64:
         return [