agent-pack-haystack 0.0.1__py3-none-any.whl

haystack_integrations/agent_pack/deep_research/__init__.py

@@ -0,0 +1,7 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+from .agent import create_deep_research_agent
+
+__all__ = ["create_deep_research_agent"]

haystack_integrations/agent_pack/deep_research/agent.py

@@ -0,0 +1,192 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+# The two agents and the composition that wires them into one deep research Agent.
+#
+# - researcher: a reusable sub-agent that researches ONE sub-question in its own isolated context and returns a
+#   compressed, cited summary.
+# - deep research agent:
+#     - before starting the Agent loop, turns the user query into a research brief using the Scope Hook.
+#     - delegates sub-questions to the researcher and accumulates the returned summaries in shared `State`.
+#     - after the Agent loop, turns the brief plus collected notes into the final report using the Write Hook.
+
+from haystack import logging
+from haystack.components.agents import Agent
+from haystack.components.builders import ChatPromptBuilder
+from haystack.components.generators.chat import OpenAIResponsesChatGenerator
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.dataclasses import ChatMessage
+from haystack.lazy_imports import LazyImport
+from haystack.tools import ComponentTool
+
+from haystack_integrations.agent_pack.deep_research import prompts
+from haystack_integrations.agent_pack.deep_research.hooks import ScopeHook, WriteHook
+from haystack_integrations.agent_pack.deep_research.tools import make_read_url_tool, think_tool
+
+with LazyImport(message="Run 'pip install tavily-haystack'") as tavily_import:
+    from haystack_integrations.tools.tavily import TavilyWebSearchTool
+
+logger = logging.getLogger(__name__)
+
+_DEFAULT_TIMEOUT = 180.0
+_DEFAULT_MAX_RETRIES = 5
+
+
+def _default_llm(model: str) -> OpenAIResponsesChatGenerator:
+    """A default OpenAI generator for a stage, with the pack's timeout/retry settings."""
+    return OpenAIResponsesChatGenerator(model=model, timeout=_DEFAULT_TIMEOUT, max_retries=_DEFAULT_MAX_RETRIES)
+
+
+def _collect_note(current: list[str] | None, message: ChatMessage) -> list[str]:
+    """State handler: append a researcher's final summary (text) to `notes`."""
+    current = current or []
+    note = getattr(message, "text", None)  # message is the researcher's last ChatMessage
+    if not note:
+        logger.info("sub-researcher returned an empty note (likely hit its step cap); skipping")
+        return current
+    logger.info(
+        "note collected (#{index}, {length} chars):\n{note}", index=len(current) + 1, length=len(note), note=note
+    )
+    return [*current, note]
+
+
+def _make_researcher_agent(
+    *,
+    researcher_llm: ChatGenerator,
+    summarizer_llm: ChatGenerator,
+    max_researcher_steps: int,
+    max_search_results: int,
+    max_content_length: int,
+) -> Agent:
+    """
+    The reusable sub-researcher.
+
+    :param researcher_llm: LLM that drives the search/read/think loop and writes the summary.
+    :param summarizer_llm: LLM used inside the `read_url` tool to summarize a page toward the question.
+    :param max_researcher_steps: Cap on the researcher's agent loop (search/read/think + final summary).
+    :param max_search_results: Results returned per `web_search` call.
+    :param max_content_length: Raw page chars fed to the summarizer (pre-summarization cap).
+    """
+    tavily_import.check()
+    return Agent(
+        chat_generator=researcher_llm,
+        system_prompt=prompts.RESEARCHER_TEMPLATE,
+        tools=[
+            TavilyWebSearchTool(top_k=max_search_results),
+            make_read_url_tool(summarizer_llm=summarizer_llm, max_content_length=max_content_length),
+            think_tool,
+        ],
+        exit_conditions=["text"],  # exits when it writes its summary
+        max_agent_steps=max_researcher_steps,
+    )
+
+
+# Minimal input schema for the research_subtopic tool
+_RESEARCH_SUBTOPIC_PARAMS = {
+    "type": "object",
+    "properties": {
+        "messages": {
+            "type": "array",
+            "description": "Exactly one user message whose content is the sub-question to research.",
+            "items": {
+                "type": "object",
+                "properties": {
+                    "role": {"type": "string", "enum": ["user"]},
+                    "content": {"type": "string", "description": "The sub-question to research."},
+                },
+                "required": ["role", "content"],
+            },
+        }
+    },
+    "required": ["messages"],
+}
+
+
+def create_deep_research_agent(
+    *,
+    scope_llm: ChatGenerator | None = None,
+    orchestrator_llm: ChatGenerator | None = None,
+    researcher_llm: ChatGenerator | None = None,
+    summarizer_llm: ChatGenerator | None = None,
+    writer_llm: ChatGenerator | None = None,
+    max_subtopics: int = 5,
+    max_concurrent_researchers: int = 5,
+    max_orchestrator_steps: int = 8,
+    max_researcher_steps: int = 20,
+    max_search_results: int = 10,
+    max_content_length: int = 50_000,
+) -> Agent:
+    """
+    Create the deep research agent.
+
+    :param scope_llm: LLM that rewrites the user query into a focused research brief.
+        Defaults to `OpenAIResponsesChatGenerator("gpt-5.4")`.
+    :param orchestrator_llm: LLM that plans the investigation and delegates the sub-questions.
+        Defaults to `OpenAIResponsesChatGenerator("gpt-5.4")`.
+    :param researcher_llm: LLM that drives each sub-researcher's search/read/think loop.
+        Defaults to `OpenAIResponsesChatGenerator("gpt-5.4-mini")`.
+    :param summarizer_llm: LLM used inside the `read_url` tool to summarize a fetched page toward
+        the question. Defaults to `OpenAIResponsesChatGenerator("gpt-5.4-mini")`.
+    :param writer_llm: LLM that turns the brief plus collected notes into the final report.
+        Defaults to `OpenAIResponsesChatGenerator("gpt-5.4")`.
+    :param max_subtopics: Maximum number of sub-questions the orchestrator may delegate (breadth).
+    :param max_concurrent_researchers: Maximum number of sub-researchers that run at the same time.
+    :param max_orchestrator_steps: Maximum steps for the orchestrator's agent loop (reflect -> delegate rounds).
+    :param max_researcher_steps: Maximum steps for each sub-researcher's agent loop.
+    :param max_search_results: Number of results returned per `web_search` call.
+    :param max_content_length: Maximum raw page characters fed to the summarizer, before summarization.
+    :returns: The deep research `Agent`. Call it with the question as a user message,
+        `agent.run(messages=[ChatMessage.from_user(question)])`; it returns a dict whose main output is
+        `report` (the final markdown report, a `str`). The dict also carries the intermediate `brief`
+        (`str`) and `notes` (`list[str]`), plus the standard Agent outputs `messages`, `last_message`,
+        `step_count`, `token_usage` and `tool_call_counts`.
+    """
+    scope_llm = scope_llm or _default_llm("gpt-5.4")
+    orchestrator_llm = orchestrator_llm or _default_llm("gpt-5.4")
+    researcher_llm = researcher_llm or _default_llm("gpt-5.4-mini")
+    summarizer_llm = summarizer_llm or _default_llm("gpt-5.4-mini")
+    writer_llm = writer_llm or _default_llm("gpt-5.4")
+
+    researcher = _make_researcher_agent(
+        researcher_llm=researcher_llm,
+        summarizer_llm=summarizer_llm,
+        max_researcher_steps=max_researcher_steps,
+        max_search_results=max_search_results,
+        max_content_length=max_content_length,
+    )
+
+    research_subtopic = ComponentTool(
+        component=researcher,
+        name="research_subtopic",
+        description=(
+            "Research ONE focused sub-question in an isolated context. "
+            "Pass the sub-question as a user message. "
+            "Returns a compressed, cited summary of the findings."
+        ),
+        parameters=_RESEARCH_SUBTOPIC_PARAMS,
+        outputs_to_string={"source": "last_message"},
+        outputs_to_state={"notes": {"source": "last_message", "handler": _collect_note}},
+    )
+
+    scope = ScopeHook(
+        generator=scope_llm,
+        prompt_builder=ChatPromptBuilder(template=prompts.SCOPE_TEMPLATE, required_variables=["query"]),
+    )
+    write = WriteHook(
+        generator=writer_llm,
+        prompt_builder=ChatPromptBuilder(template=prompts.WRITER_TEMPLATE, required_variables=["replies", "notes"]),
+    )
+
+    system_prompt = prompts.ORCHESTRATOR_TEMPLATE.replace("{{ max_subtopics }}", str(max_subtopics))
+
+    return Agent(
+        chat_generator=orchestrator_llm,
+        system_prompt=system_prompt,
+        tools=[research_subtopic, think_tool],
+        exit_conditions=["text"],
+        max_agent_steps=max_orchestrator_steps,
+        tool_concurrency_limit=max_concurrent_researchers,
+        state_schema={"notes": {"type": list}, "brief": {"type": str}, "report": {"type": str}},
+        hooks={"before_run": [scope], "after_run": [write]},
+    )

haystack_integrations/agent_pack/deep_research/hooks.py

@@ -0,0 +1,96 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""The SCOPE and WRITE stages, as serializable Agent hooks."""
+
+from haystack import logging
+from haystack.components.agents.state import State
+from haystack.components.builders import ChatPromptBuilder
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.core.serialization import default_from_dict, default_to_dict
+from haystack.dataclasses import ChatMessage
+
+logger = logging.getLogger(__name__)
+
+
+class ScopeHook:
+    """Turn the user query into a research brief, once. Runs as a `before_run` hook."""
+
+    allowed_hook_points = ("before_run",)
+
+    def __init__(self, generator: ChatGenerator, prompt_builder: ChatPromptBuilder) -> None:
+        self.generator = generator
+        self.prompt_builder = prompt_builder
+
+    def to_dict(self) -> dict:
+        """
+        Serialize the hook to a dictionary.
+
+        :returns: Dictionary with serialized data.
+        """
+        return default_to_dict(self, generator=self.generator, prompt_builder=self.prompt_builder)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "ScopeHook":
+        """
+        Deserialize the hook from a dictionary.
+
+        :param data: Dictionary to deserialize from.
+        :returns: Deserialized hook.
+        """
+        return default_from_dict(cls, data)
+
+    def run(self, state: State) -> None:
+        """Rewrite the query into a research brief and seed it as the sole user message."""
+        prompt = self.prompt_builder.run(query=state.data["messages"][-1].text)["prompt"]
+        brief = self.generator.run(messages=prompt)["replies"][0].text
+        logger.info("brief:\n{brief}", brief=brief)
+        state.set("brief", brief)
+        state.set("messages", [ChatMessage.from_user(brief)])
+
+
+class WriteHook:
+    """Write the final report from the brief and notes, once. Runs as an `after_run` hook."""
+
+    allowed_hook_points = ("after_run",)
+
+    def __init__(self, generator: ChatGenerator, prompt_builder: ChatPromptBuilder) -> None:
+        self.generator = generator
+        self.prompt_builder = prompt_builder
+
+    def to_dict(self) -> dict:
+        """
+        Serialize the hook to a dictionary.
+
+        :returns: Dictionary with serialized data.
+        """
+        return default_to_dict(self, generator=self.generator, prompt_builder=self.prompt_builder)
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "WriteHook":
+        """
+        Deserialize the hook from a dictionary.
+
+        :param data: Dictionary to deserialize from.
+        :returns: Deserialized hook.
+        """
+        return default_from_dict(cls, data)
+
+    def run(self, state: State) -> None:
+        """Write the final report from the brief and collected notes."""
+        subtopics = [
+            tc.arguments["messages"][0]["content"]
+            for m in state.data.get("messages", [])
+            for tc in (m.tool_calls or [])
+            if tc.tool_name == "research_subtopic"
+        ]
+        for i, subtopic in enumerate(subtopics, 1):
+            logger.info("subtopic #{index} delegated: {subtopic}", index=i, subtopic=subtopic)
+        notes = [n for n in state.get("notes", []) if n]
+        prompt = self.prompt_builder.run(replies=[ChatMessage.from_assistant(state.get("brief", ""))], notes=notes)[
+            "prompt"
+        ]
+        report = self.generator.run(messages=prompt)["replies"][0].text
+        state.set("report", report)
+        state.set("messages", [ChatMessage.from_assistant(report)])

haystack_integrations/agent_pack/deep_research/prompts.py

@@ -0,0 +1,104 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+SCOPE_TEMPLATE = """{% message role="system" %}
+You turn a user's research request into a clear, focused research brief.
+Today's date is {% now 'local', '%d %B %Y' %}.
+
+In 3-5 sentences, restate:
+- what exactly should be investigated,
+- the key sub-dimensions a complete answer must cover,
+- what a good answer looks like.
+
+Be specific and neutral. Do NOT invent constraints or assume facts the user did not state; where
+the request is open-ended, keep the dimension open rather than narrowing it. Write the brief in the
+same language as the user's request. Output ONLY the brief.
+{% endmessage %}
+{% message role="user" %}
+{{ query }}
+{% endmessage %}"""
+
+
+RESEARCHER_TEMPLATE = """{% message role="system" %}
+You are a focused research assistant. You are given ONE specific sub-question.
+Today's date is {% now 'local', '%d %B %Y' %}. Use it to interpret "current", "latest", and relative dates.
+Your job is to gather evidence and return a compressed, self-contained summary.
+
+Process:
+1. Use `web_search` to find relevant sources (2-3 searches is usually enough).
+2. Use `read_url` only when a snippet is promising but too shallow.
+3. Use `think_tool` after searching to assess: what did I learn? what is missing? should I stop?
+4. When you have enough evidence, STOP calling tools and write your final summary.
+
+Your final message MUST be:
+- A concise summary (a few short paragraphs) of the findings for this sub-question, with inline
+  citations using the EXACT full URLs from the search results (copied verbatim, in parentheses).
+- Followed by a "Sources:" list of every exact URL you used.
+
+Copy URLs verbatim from the search results. NEVER shorten a URL to its domain and never invent one.
+Only state what the sources support.
+{% endmessage %}"""
+
+
+ORCHESTRATOR_TEMPLATE = """{% message role="system" %}
+You are a research orchestrator. You are given a research brief.
+Today's date is {% now 'local', '%d %B %Y' %}.
+
+Process:
+1. Break the brief into at most {{ max_subtopics }} focused sub-questions, each covering a DISTINCT
+   facet of the brief. No two sub-questions may overlap; together they should partition the brief's
+   key dimensions.
+2. Call `research_subtopic` ONCE PER sub-question, emitting all the calls together in a single
+   turn so they run in parallel.
+3. When the results return, use `think_tool` to check coverage against the brief. Only if a real
+   dimension is still UNADDRESSED, do at most ONE more round — researching ONLY the genuine gap.
+   Never re-ask, rephrase, or "double-check" a sub-question that was already researched.
+4. When coverage is sufficient, STOP and reply with a one-line note that research is complete.
+   (The detailed findings are already collected — do not repeat them.)
+
+Stay tight: never exceed {{ max_subtopics }} sub-questions in the first round.
+{% endmessage %}"""
+
+
+SUMMARIZE_TEMPLATE = """{% message role="user" %}
+You are extracting information from a web page for a research task.
+
+Research question:
+{{ question }}
+
+Web page content:
+{{ documents[0].content[:__MAXLEN__] }}
+
+Write a concise, factual summary of ONLY the parts of the page relevant to the research question.
+Preserve key facts, numbers, names, dates and short quotes verbatim. If the page contains nothing
+relevant, say so in one line. Do not add anything not present in the page.
+{% endmessage %}"""
+
+
+WRITER_TEMPLATE = """{% message role="system" %}
+Today's date is {% now 'local', '%d %B %Y' %}. Write the report in the same language as the brief.
+{% endmessage %}
+{% message role="user" %}
+You are a research report writer. Using ONLY the collected research notes below,
+write a well-structured markdown report that answers the brief.
+
+Requirements:
+- Open with a one-paragraph executive summary.
+- Organise the body into clear sections with markdown headings.
+- Be accurate and concise. Do not add facts that are not in the notes.
+- Cite sources INLINE as Markdown links: `[short descriptive text](EXACT-URL-from-notes)`. Put a
+  citation on every sentence that states a fact from the notes.
+- Use ONLY the EXACT full URLs that appear in the notes, copied verbatim. NEVER shorten a URL to its
+  domain and never invent one.
+- End with a "## Sources" section listing the exact URLs you cited.
+
+# Brief
+{{ replies[0].text }}
+
+# Research notes
+{% for note in notes %}
+---
+{{ note }}
+{% endfor %}
+{% endmessage %}"""

haystack_integrations/agent_pack/deep_research/tools.py

@@ -0,0 +1,113 @@
+# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
+#
+# SPDX-License-Identifier: Apache-2.0
+
+"""
+The tools defined here for the agents.
+
+- `read_url`: a `PipelineTool` wrapping a small summarization pipeline. The fetched
+  page is routed by MIME type (`FileTypeRouter`) to `HTMLToDocument` or
+  `PyPDFToDocument` so PDFs are parsed too, then summarized toward the question.
+- `think_tool`: the reflection no-op — a plain `@tool`.
+
+`web_search` is the `TavilyWebSearchTool` provided by the Tavily integration; it is wired in `agent.py`.
+"""
+
+from typing import Annotated
+
+from haystack import Document, Pipeline, component, logging
+from haystack.components.builders import ChatPromptBuilder
+from haystack.components.converters import HTMLToDocument, PyPDFToDocument
+from haystack.components.fetchers import LinkContentFetcher
+from haystack.components.generators.chat.types import ChatGenerator
+from haystack.components.joiners import DocumentJoiner
+from haystack.components.routers import FileTypeRouter
+from haystack.tools import PipelineTool, tool
+
+from haystack_integrations.agent_pack.deep_research import prompts
+
+logger = logging.getLogger(__name__)
+
+
+@component
+class ContentGate:
+    """Content gate for `read_url`: pass documents to the summarizer only when the page actually has text."""
+
+    @component.output_types(documents=list[Document], empty=bool)
+    def run(self, documents: list[Document]) -> dict:
+        """
+        Forward documents to the summarizer only when the fetched page has readable text.
+
+        :param documents: Documents produced from the fetched page.
+        :returns: `{"documents": [...]}` when the page has text, otherwise `{"empty": True}`.
+        """
+        if documents and documents[0].content:
+            return {"documents": documents}
+        return {"empty": True}
+
+
+def _read_url_result(result: dict) -> str:
+    """
+    Turn the `read_url` pipeline result into one line for the agent (an `outputs_to_string` handler).
+
+    We don't treat HTTP/network failures here: the `LinkContentFetcher` has `raise_on_failure=True`, so these
+    errors are sent to the agent wrapped in an error tool-result message.
+    """
+    replies = result.get("replies")
+    if replies and replies[0].text:
+        return replies[0].text
+
+    unsupported = result.get("unclassified")
+    if unsupported:
+        content_type = unsupported[0].meta.get("content_type", "unknown")
+        return f"Page not read: unsupported content type {content_type}; only HTML and PDF are read."
+
+    return "Page not read: the page returned no readable text."
+
+
+def make_read_url_tool(*, summarizer_llm: ChatGenerator, max_content_length: int) -> PipelineTool:
+    """
+    `read_url` = PipelineTool over a fetch -> convert to text -> summarize-based-on-question pipeline.
+
+    :param summarizer_llm: LLM that summarizes the fetched page toward the agent's question.
+    :param max_content_length: Raw page chars fed to the summarizer (pre-summarization cap).
+    """
+    template = prompts.SUMMARIZE_TEMPLATE.replace("__MAXLEN__", str(max_content_length))
+
+    pipe = Pipeline()
+    pipe.add_component("fetcher", LinkContentFetcher(raise_on_failure=True, retry_attempts=2, timeout=10))
+    pipe.add_component("router", FileTypeRouter(mime_types=["text/html", "application/pdf"]))
+    pipe.add_component("html", HTMLToDocument())
+    pipe.add_component("pdf", PyPDFToDocument())
+    pipe.add_component("joiner", DocumentJoiner())
+    pipe.add_component("content_gate", ContentGate())
+    pipe.add_component("builder", ChatPromptBuilder(template=template, required_variables=["question", "documents"]))
+    pipe.add_component("summarizer", summarizer_llm)
+    pipe.connect("fetcher.streams", "router.sources")
+    pipe.connect("router.text/html", "html.sources")
+    pipe.connect("router.application/pdf", "pdf.sources")
+    pipe.connect("html.documents", "joiner.documents")
+    pipe.connect("pdf.documents", "joiner.documents")
+    pipe.connect("joiner.documents", "content_gate.documents")
+    pipe.connect("content_gate.documents", "builder.documents")
+    pipe.connect("builder.prompt", "summarizer.messages")
+
+    return PipelineTool(
+        pipeline=pipe,
+        name="read_url",
+        description=(
+            "Fetch a web page or PDF and get a summary focused on your question. Provide the URL to "
+            "read and the specific question you want answered from it. Use when a snippet is too shallow."
+        ),
+        input_mapping={"urls": ["fetcher.urls"], "question": ["builder.question"]},
+        outputs_to_string={"handler": _read_url_result},
+    )
+
+
+@tool
+def think_tool(
+    reflection: Annotated[str, "What you learned, what is still missing, and whether to keep searching or stop."],
+) -> str:
+    """Record a brief reflection to plan your research. Call this between searches."""
+    logger.info("think: {reflection}", reflection=reflection)
+    return "Reflection noted"