deepset-mcp 0.0.3rc1__py3-none-any.whl → 0.0.4__py3-none-any.whl

deepset_mcp/benchmark/runner/streaming.py

@@ -1,360 +0,0 @@
-"""
-Async-compatible practical streaming callback for deepset agent responses.
-
-Handles text streaming, tool calls, and tool results with nice console formatting.
-"""
-
-import json
-from typing import Any
-
-from haystack.dataclasses.streaming_chunk import StreamingChunk
-from rich.console import Console
-from rich.live import Live
-from rich.markdown import Markdown
-
-
-class StreamingCallbackManager:
-    """
-    Async-compatible callback tailored to your exact streaming structure.
-
-    Handles the specific patterns from your deepset agent.
-    """
-
-    def __init__(self) -> None:
-        """Initialize the streaming callback."""
-        self.console = Console()
-        self.active_tools: dict[int, dict[str, Any]] = {}
-        self.accumulated_text = ""
-        self.live_display: Live | None = None
-        self.text_started = False
-
-    async def __call__(self, chunk: StreamingChunk) -> None:
-        """Process each streaming chunk asynchronously."""
-        await self._handle_chunk(chunk)
-
-    async def _handle_chunk(self, chunk: StreamingChunk) -> None:
-        """Handle different types of chunks based on your data structure."""
-        meta = chunk.meta
-
-        # 1. Handle text streaming (like "I'll help you troubleshoot...")
-        if self._is_text_delta(meta):
-            text = meta["delta"]["text"]
-            self.accumulated_text += text
-            await self._render_markdown_optimistic()
-
-        # 2. Handle tool call start (like list_pipelines, get_pipeline)
-        elif self._is_tool_start(meta):
-            await self._handle_tool_start(meta)
-
-        # 3. Handle tool arguments streaming (partial JSON)
-        elif self._is_tool_args(meta):
-            await self._handle_tool_args(meta)
-
-        # 4. Handle tool results
-        elif self._is_tool_result(meta):
-            await self._handle_tool_result(meta)
-
-        # 5. Handle message deltas (usage info, etc.)
-        elif self._is_message_delta(meta):
-            await self._handle_message_delta(meta)
-
-        if self._is_finish_event(meta):
-            await self._handle_finish_event(meta)
-
-    async def _render_markdown_optimistic(self) -> None:
-        """Render accumulated text as markdown optimistically."""
-        if not self.accumulated_text.strip():
-            return
-
-        try:
-            # Attempt to render as markdown
-            markdown = Markdown(self.accumulated_text)
-
-            # Start live display if not already started
-            if not self.live_display:
-                self.live_display = Live(markdown, console=self.console, refresh_per_second=10)
-                self.live_display.start()
-                self.text_started = True
-            else:
-                # Update the live display
-                self.live_display.update(markdown)
-
-        except Exception:
-            # Fallback to plain text if markdown parsing fails
-            if not self.live_display:
-                self.live_display = Live(self.accumulated_text, console=self.console, refresh_per_second=10)
-                self.live_display.start()
-                self.text_started = True
-            else:
-                self.live_display.update(self.accumulated_text)
-
-    def _is_text_delta(self, meta: dict[str, Any]) -> bool:
-        """Check if this is a text streaming chunk."""
-        return meta.get("type") == "content_block_delta" and meta.get("delta", {}).get("type") == "text_delta"
-
-    def _is_tool_start(self, meta: dict[str, Any]) -> bool:
-        """Check if this is the start of a tool call."""
-        return meta.get("type") == "content_block_start" and meta.get("content_block", {}).get("type") == "tool_use"
-
-    def _is_tool_args(self, meta: dict[str, Any]) -> bool:
-        """Check if this is tool arguments streaming."""
-        return meta.get("type") == "content_block_delta" and meta.get("delta", {}).get("type") == "input_json_delta"
-
-    def _is_tool_result(self, meta: dict[str, Any]) -> bool:
-        """Check if this is a tool result."""
-        return "tool_result" in meta and "tool_call" in meta
-
-    def _is_message_delta(self, meta: dict[str, Any]) -> bool:
-        """Check if this is a message-level delta."""
-        return meta.get("type") == "message_delta"
-
-    def _is_finish_event(self, meta: dict[str, Any]) -> bool:
-        """Check if this is a finish event."""
-        return "stop_reason" in meta.get("delta", {})
-
-    async def _handle_tool_start(self, meta: dict[str, Any]) -> None:
-        """Handle the start of a tool call."""
-        content_block = meta["content_block"]
-        tool_name = content_block["name"]
-        tool_id = content_block["id"]
-        index = meta["index"]
-
-        # Stop live display if active
-        if self.live_display:
-            self.live_display.stop()
-            self.live_display = None
-
-        # Store tool state
-        self.active_tools[index] = {
-            "name": tool_name,
-            "id": tool_id,
-            "args_json": "",
-            "started": True,
-            "args_displayed": False,
-        }
-
-        # Display tool call header (text accumulation continues after tools)
-        self.console.print()  # New line
-        self.console.print("┌─ 🔧 Tool Call", style="bold cyan")
-        self.console.print(f"│ Name: {tool_name}", style="cyan")
-
-    async def _handle_tool_args(self, meta: dict[str, Any]) -> None:
-        """Handle streaming tool arguments."""
-        index = meta["index"]
-        if index not in self.active_tools:
-            return
-
-        partial_json = meta["delta"]["partial_json"]
-        self.active_tools[index]["args_json"] += partial_json
-
-        # Try to show current args when we have complete JSON
-        await self._try_display_complete_args(index)
-
-    async def _try_display_complete_args(self, index: int) -> None:
-        """Try to display complete arguments when JSON is valid."""
-        tool = self.active_tools[index]
-
-        try:
-            # Try to parse the current JSON
-            if tool["args_json"].strip() and not tool["args_displayed"]:
-                args = json.loads(tool["args_json"])
-
-                # Display arguments in multi-line format
-                await self._display_tool_arguments(args)
-                tool["args_displayed"] = True
-
-        except json.JSONDecodeError:
-            # Still accumulating JSON, wait for more
-            pass
-
-    async def _display_tool_arguments(self, args: dict[str, Any]) -> None:
-        """Display tool arguments in a pretty multi-line format."""
-        if not args:
-            self.console.print("│ (no arguments)", style="dim")
-            return
-
-        self.console.print("│ Arguments:", style="cyan")
-
-        for arg_name, arg_value in args.items():
-            self.console.print(f"│   {arg_name}:", style="yellow")
-
-            # Format the argument value with line limit
-            formatted_value = await self._format_argument_value(arg_value, max_lines=5)
-
-            # Display each line of the value with proper indentation
-            for line in formatted_value:
-                self.console.print(f"│     {line}", style="white")
-
-    async def _format_argument_value(self, value: Any, max_lines: int = 5) -> list[str]:
-        """Format an argument value with line limits."""
-        if value is None:
-            return ["null"]
-
-        if isinstance(value, bool):
-            return [str(value).lower()]
-
-        if isinstance(value, int | float):
-            return [str(value)]
-
-        if isinstance(value, str):
-            # Handle multi-line strings
-            lines = value.split("\n")
-
-            # Limit lines
-            display_lines = lines[:max_lines]
-            result = []
-
-            for line in display_lines:
-                # Wrap long lines at 60 characters for readability
-                if len(line) <= 60:
-                    result.append(f'"{line}"' if line else '""')
-                else:
-                    result.append(f'"{line[:57]}..."')
-
-            # Add truncation indicator if needed
-            if len(lines) > max_lines:
-                result.append(f"... ({len(lines) - max_lines} more lines)")
-
-            return result
-
-        if isinstance(value, list | dict):
-            # Pretty print complex objects
-            try:
-                json_str = json.dumps(value, indent=2)
-                lines = json_str.split("\n")
-
-                display_lines = lines[:max_lines]
-                if len(lines) > max_lines:
-                    display_lines.append(f"... ({len(lines) - max_lines} more lines)")
-
-                return display_lines
-            except Exception:
-                return [str(value)[:100] + "..." if len(str(value)) > 100 else str(value)]
-
-        # Fallback for other types
-        str_value = str(value)
-        if len(str_value) > 60:
-            return [str_value[:57] + "..."]
-        return [str_value]
-
-    async def _handle_tool_result(self, meta: dict[str, Any]) -> None:
-        """Handle tool execution results."""
-        tool_result = meta["tool_result"]
-
-        # Close the tool call display
-        self.console.print("└─ ✅ Completed", style="green")
-
-        # Display tool result content (max 10 lines)
-        if tool_result:
-            await self._display_tool_result(tool_result)
-
-    async def _display_tool_result(self, tool_result: str, max_lines: int = 10) -> None:
-        """Display tool result with a maximum number of lines."""
-        try:
-            # Parse the tool result JSON
-            if isinstance(tool_result, str):
-                result_data = json.loads(tool_result)
-
-                # Extract the actual content
-                content_text = await self._extract_result_content(result_data)
-
-                if content_text:
-                    # Split into lines and limit to max_lines
-                    lines = content_text.split("\n")
-                    display_lines = lines[:max_lines]
-
-                    # Show the result with indentation
-                    self.console.print("  ┌─ Result:", style="dim cyan")
-                    for line in display_lines:
-                        if line.strip():  # Only show non-empty lines
-                            self.console.print(f"  │ {line}", style="dim")
-
-                    # Show truncation indicator if needed
-                    if len(lines) > max_lines:
-                        remaining = len(lines) - max_lines
-                        self.console.print(f"  │ ... ({remaining} more lines)", style="dim yellow")
-
-                    self.console.print("  └─", style="dim cyan")
-                else:
-                    self.console.print("  → Result received", style="dim green")
-
-        except Exception:
-            # Fallback for unparseable results
-            self.console.print("  → Result received", style="dim green")
-
-    async def _extract_result_content(self, result_data: dict[str, Any]) -> str | None:
-        """Extract meaningful content from tool result data."""
-        try:
-            # Handle the specific structure from your deepset results
-            if isinstance(result_data, dict):
-                content = result_data.get("content", [])
-
-                if isinstance(content, list) and content:
-                    # Get the first content item
-                    first_content = content[0]
-
-                    if isinstance(first_content, dict):
-                        text_content = first_content.get("text", "")
-
-                        # Handle nested JSON strings (like "@obj_001 → deepset_mcp...")
-                        if text_content.startswith('"') and text_content.endswith('"'):
-                            # Parse the inner JSON string
-                            inner_content = json.loads(text_content)
-                            formatted = await self._format_deepset_content(str(inner_content))
-                            return formatted
-                        else:
-                            return str(text_content) if text_content else None
-
-            return str(result_data) if result_data else None
-
-        except Exception:
-            return str(result_data) if result_data else None
-
-    async def _format_deepset_content(self, content: str) -> str:
-        """Format deepset-specific content for better readability."""
-        try:
-            # Handle content like "@obj_001 → deepset_mcp.api.pipeline.models.PipelineList..."
-            if " → " in content:
-                parts = content.split(" → ", 1)
-                if len(parts) == 2:
-                    obj_id, obj_content = parts
-
-                    # Clean up the object content for better display
-                    formatted = obj_content.replace("\\n", "\n").replace("\\\\", "\\")
-
-                    # Add object ID as header
-                    return f"Object: {obj_id}\n{formatted}"
-
-            # Fallback: clean up escape sequences
-            return content.replace("\\n", "\n").replace("\\\\", "\\")
-
-        except Exception:
-            return content
-
-    async def _handle_message_delta(self, meta: dict[str, Any]) -> None:
-        """Handle message-level information."""
-        delta = meta.get("delta", {})
-
-        # Could show usage info if desired
-        if "usage" in delta:
-            usage = delta["usage"]
-            if usage.get("output_tokens"):
-                # Optionally show token usage
-                pass
-
-    async def _handle_finish_event(self, meta: dict[str, Any]) -> None:
-        """Handle finish events."""
-        finish_reason = meta.get("delta", {}).get("stop_reason")
-        if finish_reason == "tool_call_results":
-            # Clean up after tool calls
-            self.active_tools.clear()
-            self.console.print()  # Extra line after tools
-        elif finish_reason == "end_turn":
-            # Stop live display and reset for next interaction
-            if self.live_display:
-                self.live_display.stop()
-                self.live_display = None
-                # Ensure cursor is on a new line for the next prompt
-                self.console.print()
-            self.accumulated_text = ""
-            self.text_started = False

deepset_mcp/benchmark/runner/teardown_actions.py

@@ -1,196 +0,0 @@
-import asyncio
-import os
-from typing import Any
-
-from deepset_mcp.api.client import AsyncDeepsetClient
-from deepset_mcp.benchmark.runner.models import TestCaseConfig
-
-
-def _get_api_key(explicit_key: str | None) -> str:
-    """
-    Return whichever API key to use: explicit_key takes precedence, otherwise read DP_API_KEY from the environment.
-
-    If still missing, raise ValueError.
-    """
-    if explicit_key:
-        return explicit_key
-    env_key = os.getenv("DP_API_KEY")
-    if not env_key:
-        raise ValueError("No API key provided: pass --api-key or set DP_API_KEY in env.")
-    return env_key
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# 1) LOW-LEVEL: "teardown_pipeline" and "teardown_index" using AsyncDeepsetClient as a context manager
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-async def teardown_pipeline_async(
-    *,
-    pipeline_name: str,
-    workspace_name: str,
-    api_key: str | None = None,
-) -> None:
-    """
-    Delete a pipeline in the given workspace.
-
-    Uses DP_API_KEY or explicit api_key.
-    """
-    key_to_use = _get_api_key(api_key)
-    async with AsyncDeepsetClient(api_key=key_to_use) as client:
-        await client.pipelines(workspace=workspace_name).delete(pipeline_name)
-    return None
-
-
-async def teardown_index_async(
-    *,
-    index_name: str,
-    workspace_name: str,
-    api_key: str | None = None,
-) -> None:
-    """
-    Delete an index in the given workspace.
-
-    Uses DP_API_KEY or explicit api_key.
-    """
-    key_to_use = _get_api_key(api_key)
-    async with AsyncDeepsetClient(api_key=key_to_use) as client:
-        await client.indexes(workspace=workspace_name).delete(index_name)
-    return None
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# 2) MID-LEVEL: teardown a full test-case (pipeline + index if present)
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-async def teardown_test_case_async(
-    *,
-    test_cfg: TestCaseConfig,
-    workspace_name: str,
-    api_key: str | None = None,
-) -> None:
-    """
-    Given a TestCaseConfig, delete the index and the pipeline in the specified workspace.
-
-    Uses DP_API_KEY or explicit api_key.
-    """
-    # 1) If there's a "query pipeline" to delete:
-    if test_cfg.query_yaml:
-        assert test_cfg.query_name is not None  # already validated by Pydantic model; added to satisfy mypy
-        await teardown_pipeline_async(
-            pipeline_name=test_cfg.query_name,
-            workspace_name=workspace_name,
-            api_key=api_key,
-        )
-
-    # 2) If there's an index to delete:
-    if test_cfg.index_yaml:
-        assert test_cfg.index_name is not None  # already validated by Pydantic model; added to satisfy mypy
-        await teardown_index_async(
-            index_name=test_cfg.index_name,
-            workspace_name=workspace_name,
-            api_key=api_key,
-        )
-
-    return None
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# 3) HIGH-LEVEL: parallel "teardown all" with configurable concurrency
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-async def teardown_all_async(
-    *,
-    test_cfgs: list[TestCaseConfig],
-    workspace_name: str,
-    api_key: str | None = None,
-    concurrency: int = 5,
-) -> None:
-    """
-    Given a list of TestCaseConfig, delete all indexes and pipelines in parallel.
-
-    Uses DP_API_KEY or explicit api_key.
-    """
-    semaphore = asyncio.Semaphore(concurrency)
-    tasks: list[asyncio.Task[Any]] = []
-
-    async def sem_task(cfg: TestCaseConfig) -> str:
-        async with semaphore:
-            await teardown_test_case_async(test_cfg=cfg, workspace_name=workspace_name, api_key=api_key)
-            return cfg.name
-
-    for cfg in test_cfgs:
-        tasks.append(asyncio.create_task(sem_task(cfg)))
-
-    done, _ = await asyncio.wait(tasks, return_when=asyncio.ALL_COMPLETED)
-    errors: list[Exception] = []
-    for t in done:
-        if t.exception():
-            errors.append(t.exception())  # type: ignore
-
-    if errors:
-        raise RuntimeError(f"Errors during teardown: {errors}")
-
-    return None
-
-
-# ─────────────────────────────────────────────────────────────────────────────
-# 4) SYNC WRAPPERS for all of the above (now accept api_key)
-# ─────────────────────────────────────────────────────────────────────────────
-
-
-def teardown_pipeline(
-    *,
-    pipeline_name: str,
-    workspace_name: str,
-    api_key: str | None = None,
-) -> None:
-    """Synchronous wrapper for teardown_pipeline_async. Blocks until the pipeline is deleted."""
-    return asyncio.run(
-        teardown_pipeline_async(
-            pipeline_name=pipeline_name,
-            workspace_name=workspace_name,
-            api_key=api_key,
-        )
-    )
-
-
-def teardown_index(
-    *,
-    index_name: str,
-    workspace_name: str,
-    api_key: str | None = None,
-) -> None:
-    """Synchronous wrapper for teardown_index_async. Blocks until the index is deleted."""
-    return asyncio.run(
-        teardown_index_async(
-            index_name=index_name,
-            workspace_name=workspace_name,
-            api_key=api_key,
-        )
-    )
-
-
-def teardown_test_case(
-    *,
-    test_cfg: TestCaseConfig,
-    workspace_name: str,
-    api_key: str | None = None,
-) -> None:
-    """Synchronous wrapper: blocks until both pipeline and index (if any) are deleted."""
-    return asyncio.run(teardown_test_case_async(test_cfg=test_cfg, workspace_name=workspace_name, api_key=api_key))
-
-
-def teardown_all(
-    *,
-    test_cfgs: list[TestCaseConfig],
-    workspace_name: str,
-    api_key: str | None = None,
-    concurrency: int = 5,
-) -> None:
-    """Synchronous wrapper for teardown_all_async. Blocks until all test-cases are deleted."""
-    return asyncio.run(
-        teardown_all_async(test_cfgs=test_cfgs, workspace_name=workspace_name, api_key=api_key, concurrency=concurrency)
-    )

deepset_mcp/benchmark/runner/tracing.py

@@ -1,21 +0,0 @@
-from haystack.tracing.tracer import enable_tracing as haystack_enable_tracing, tracer
-from haystack_integrations.tracing.langfuse import LangfuseTracer
-from langfuse import Langfuse
-
-
-def enable_tracing(
-    secret_key: str,
-    public_key: str,
-    name: str,
-) -> None:
-    """Enables tracing with langfuse."""
-    resolved_langfuse_client_kwargs = {
-        "secret_key": secret_key,
-        "public_key": public_key,
-    }
-    tracer.is_content_tracing_enabled = True
-    langfuse_tracer = LangfuseTracer(
-        tracer=Langfuse(**resolved_langfuse_client_kwargs),
-        name=name,
-    )
-    haystack_enable_tracing(langfuse_tracer)

deepset_mcp/benchmark/tasks/chat_rag_answers_wrong_format.yml

@@ -1,16 +0,0 @@
-name: "chat_rag_answers_wrong_format"
-objective: "Add an AnswerBuilder and connect it to the answers output."
-prompt: "Can you check why my chat rag pipeline doesn't work."
-query_yaml: "pipelines/chat_rag_answers_wrong_format.yml"
-query_name: "rag-chat-test"
-index_yaml: "pipelines/standard_index.yml"
-index_name: "standard-index"
-tags:
-  - "debug"
-  - "pipeline-outputs"
-judge_prompt: |
-  The Agent has:
-  - successfully added an AnswerBuilder
-  - connected the qa_llm.replies output to AnswerBuilder.replies
-  - connected the query input to the AnswerBuilder
-  - added the output of AnswerBuilder as the overall "answers" output of the pipeline

deepset_mcp/benchmark/tasks/documents_output_wrong.yml

@@ -1,13 +0,0 @@
-name: "documents_output_wrong"
-objective: "Connect the retrieved documents to the documents output."
-prompt: "Can you check why my rag pipeline doesn't work."
-query_yaml: "pipelines/rag_documents_wrong_format.yml"
-query_name: "rag"
-index_yaml: "pipelines/standard_index.yml"
-index_name: "standard-index"
-tags:
-  - "debug"
-  - "pipeline-outputs"
-judge_prompt: |
-  The Agent has:
-  - successfully connected retrieved documents to the document output

deepset_mcp/benchmark/tasks/jinja_str_instead_of_complex_type.yml

@@ -1,11 +0,0 @@
-name: "jinja_str_instead_of_complex_type"
-objective: "Add parentheses after the filter in the OutputAdapter to get correct attribute access."
-prompt: "Can you check why my chat agent pipeline doesn't work."
-query_yaml: "pipelines/chat_agent_jinja_syntax.yml"
-query_name: "chat-agent"
-tags:
-  - "debug"
-  - "jinja"
-judge_prompt: |
-  The Agent has:
-  - added parentheses around messages|last so that attribute access works correctly

deepset_mcp/benchmark/tasks/jinja_syntax_error.yml

@@ -1,11 +0,0 @@
-name: "jinja_syntax_error"
-objective: "Add double braces to the OutputAdapter to fix the syntax error."
-prompt: "Can you check why my chat agent pipeline doesn't work."
-query_yaml: "pipelines/chat_agent_jinja_str.yml"
-query_name: "chat-agent"
-tags:
-  - "debug"
-  - "jinja"
-judge_prompt: |
-  The Agent has:
-  - successfully fixed the template on the OutputAdapter to not contain additional whitespace

deepset_mcp/benchmark/tasks/missing_output_mapping.yml

@@ -1,14 +0,0 @@
-name: "missing_output_mapping"
-objective: "Add pipeline inputs and outputs."
-prompt: "Can you check why my chat rag pipeline doesn't work."
-query_yaml: "pipelines/chat_rag_missing_output_mapping.yml"
-query_name: "rag-chat-test"
-index_yaml: "pipelines/standard_index.yml"
-index_name: "standard-index"
-tags:
-  - "debug"
-  - "pipeline-outputs"
-judge_prompt: |
-  The Agent has:
-  - successfully added an outputs key
-  - connected the document and answer outputs

deepset_mcp/benchmark/tasks/no_query_input.yml

@@ -1,13 +0,0 @@
-name: "no_query_input"
-objective: "Connect a query inputs to all relevant components."
-prompt: "Can you check why my rag pipeline doesn't work."
-query_yaml: "pipelines/rag_no_query_input.yml"
-query_name: "rag"
-index_yaml: "pipelines/standard_index.yml"
-index_name: "standard-index"
-tags:
-  - "debug"
-  - "pipeline-outputs"
-judge_prompt: |
-  The Agent has:
-  - successfully connected a query input to all retrievers, ranker and Prompt- and AnswerBuilder