flock-core 0.5.0b63__py3-none-any.whl → 0.5.0b70__py3-none-any.whl

flock/agent.py

@@ -5,7 +5,7 @@ from __future__ import annotations
 import asyncio
 import os
 from dataclasses import dataclass
-from typing import TYPE_CHECKING, Any
+from typing import TYPE_CHECKING, Any, TypedDict
 
 from pydantic import BaseModel
 
@@ -27,6 +27,38 @@ if TYPE_CHECKING:  # pragma: no cover - type hints only
     from flock.orchestrator import Flock
 
 
+class MCPServerConfig(TypedDict, total=False):
+    """Configuration for MCP server assignment to an agent.
+
+    All fields are optional. If omitted, no restrictions apply.
+
+    Attributes:
+        roots: Filesystem paths this server can access.
+               Empty list or omitted = no mount restrictions.
+        tool_whitelist: Tool names the agent can use from this server.
+                       Empty list or omitted = all tools available.
+
+    Examples:
+        >>> # No restrictions
+        >>> config: MCPServerConfig = {}
+
+        >>> # Mount restrictions only
+        >>> config: MCPServerConfig = {"roots": ["/workspace/data"]}
+
+        >>> # Tool whitelist only
+        >>> config: MCPServerConfig = {"tool_whitelist": ["read_file", "write_file"]}
+
+        >>> # Both restrictions
+        >>> config: MCPServerConfig = {
+        ...     "roots": ["/workspace/data"],
+        ...     "tool_whitelist": ["read_file"]
+        ... }
+    """
+
+    roots: list[str]
+    tool_whitelist: list[str]
+
+
 @dataclass
 class AgentOutput:
     spec: ArtifactSpec
@@ -67,7 +99,7 @@ class Agent(metaclass=AutoTracedMeta):
         self.engines: list[EngineComponent] = []
         self.best_of_n: int = 1
         self.best_of_score: Callable[[EvalResult], float] | None = None
-        self.max_concurrency: int = 1
+        self.max_concurrency: int = 2
         self._semaphore = asyncio.Semaphore(self.max_concurrency)
         self.calls_func: Callable[..., Any] | None = None
         self.tools: set[Callable[..., Any]] = set()
@@ -77,6 +109,9 @@ class Agent(metaclass=AutoTracedMeta):
         self.prevent_self_trigger: bool = True  # T065: Prevent infinite feedback loops
         # MCP integration
         self.mcp_server_names: set[str] = set()
+        self.mcp_mount_points: list[str] = []  # Deprecated: Use mcp_server_mounts instead
+        self.mcp_server_mounts: dict[str, list[str]] = {}  # Server-specific mount points
+        self.tool_whitelist: list[str] | None = None
 
     @property
     def identity(self) -> AgentIdentity:
@@ -137,15 +172,30 @@ class Agent(metaclass=AutoTracedMeta):
             # Get the MCP manager from orchestrator
             manager = self._orchestrator.get_mcp_manager()
 
-            # Import tool wrapper
-
             # Fetch tools from all assigned servers
             tools_dict = await manager.get_tools_for_agent(
                 agent_id=self.name,
                 run_id=ctx.task_id,
                 server_names=self.mcp_server_names,
+                server_mounts=self.mcp_server_mounts,  # Pass server-specific mounts
             )
 
+            # Whitelisting logic
+            tool_whitelist = self.tool_whitelist
+            if (
+                tool_whitelist is not None
+                and isinstance(tool_whitelist, list)
+                and len(tool_whitelist) > 0
+            ):
+                filtered_tools: dict[str, Any] = {}
+                for tool_key, tool_entry in tools_dict.items():
+                    if isinstance(tool_entry, dict):
+                        original_name = tool_entry.get("original_name", None)
+                        if original_name is not None and original_name in tool_whitelist:
+                            filtered_tools[tool_key] = tool_entry
+
+                tools_dict = filtered_tools
+
             # Convert to DSPy tool callables
             dspy_tools = []
             for namespaced_name, tool_info in tools_dict.items():
@@ -630,30 +680,103 @@ class AgentBuilder:
         self._agent.tools.update(funcs)
         return self
 
-    def with_mcps(self, server_names: Iterable[str]) -> AgentBuilder:
-        """Assign MCP servers to this agent.
-
-        Architecture Decision: AD001 - Two-Level Architecture
-        Agents reference servers registered at orchestrator level.
-
-        Args:
-            server_names: Names of MCP servers this agent should use
-
-        Returns:
-            self for method chaining
-
-        Raises:
-            ValueError: If any server name is not registered with orchestrator
-
-        Example:
-            >>> agent = (
-            ...     orchestrator.agent("file_agent")
-            ...     .with_mcps(["filesystem", "github"])
-            ...     .build()
-            ... )
+    def with_mcps(
+        self,
+        servers: (
+            Iterable[str]
+            | dict[str, MCPServerConfig | list[str]]  # Support both new and old format
+            | list[str | dict[str, MCPServerConfig | list[str]]]
+        ),
+    ) -> AgentBuilder:
+        """Assign MCP servers to this agent with optional server-specific mount points.
+
+                Architecture Decision: AD001 - Two-Level Architecture
+                Agents reference servers registered at orchestrator level.
+
+                Args:
+                    servers: One of:
+                        - List of server names (strings) - no specific mounts
+                        - Dict mapping server names to MCPServerConfig or list[str] (backward compatible)
+                        - Mixed list of strings and dicts for flexibility
+
+                Returns:
+                    self for method chaining
+
+                Raises:
+                    ValueError: If any server name is not registered with orchestrator
+
+                Examples:
+                    >>> # Simple: no mount restrictions
+                    >>> agent.with_mcps(["filesystem", "github"])
+
+                    >>> # New format: Server-specific config with roots and tool whitelist
+                    >>> agent.with_mcps({
+                    ...     "filesystem": {"roots": ["/workspace/dir/data"], "tool_whitelist": ["read_file"]},
+                    ...     "github": {}  # No restrictions for github
+                    ... })
+
+                    >>> # Old format: Direct list (backward compatible)
+                    >>> agent.with_mcps({
+                    ...     "filesystem": ["/workspace/dir/data"],  # Old format still works
+                    ... })
+
+                    >>> # Mixed: backward compatible
+                    >>> agent.with_mcps([
+                    ...     "github",  # No mounts
+                    ...     {"filesystem": {"roots": ["mount1", "mount2"] } }
+        ```
+                    ... ])
         """
-        # Convert to set for efficient lookup
-        server_set = set(server_names)
+        # Parse input into server_names and mounts
+        server_set: set[str] = set()
+        server_mounts: dict[str, list[str]] = {}
+        whitelist = None
+
+        if isinstance(servers, dict):
+            # Dict format: supports both old and new formats
+            # Old: {"server": ["/path1", "/path2"]}
+            # New: {"server": {"roots": ["/path1"], "tool_whitelist": ["tool1"]}}
+            for server_name, server_config in servers.items():
+                server_set.add(server_name)
+
+                # Check if it's the old format (direct list) or new format (MCPServerConfig dict)
+                if isinstance(server_config, list):
+                    # Old format: direct list of paths (backward compatibility)
+                    if len(server_config) > 0:
+                        server_mounts[server_name] = list(server_config)
+                elif isinstance(server_config, dict):
+                    # New format: MCPServerConfig with optional roots and tool_whitelist
+                    mounts = server_config.get("roots", None)
+                    if mounts is not None and isinstance(mounts, list) and len(mounts) > 0:
+                        server_mounts[server_name] = list(mounts)
+
+                    config_whitelist = server_config.get("tool_whitelist", None)
+                    if (
+                        config_whitelist is not None
+                        and isinstance(config_whitelist, list)
+                        and len(config_whitelist) > 0
+                    ):
+                        whitelist = config_whitelist
+        elif isinstance(servers, list):
+            # List format: can be mixed
+            for item in servers:
+                if isinstance(item, str):
+                    # Simple server name
+                    server_set.add(item)
+                elif isinstance(item, dict):
+                    # Dict with mounts
+                    for server_name, mounts in item.items():
+                        server_set.add(server_name)
+                        if mounts:
+                            server_mounts[server_name] = list(mounts)
+                else:
+                    raise TypeError(
+                        f"Invalid server specification: {item}. "
+                        f"Expected string or dict, got {type(item).__name__}"
+                    )
+        else:
+            # Assume it's an iterable of strings (backward compatibility)
+            server_set = set(servers)
 
         # Validate all servers exist in orchestrator
         registered_servers = set(self._orchestrator._mcp_configs.keys())
@@ -669,6 +792,61 @@ class AgentBuilder:
 
         # Store in agent
         self._agent.mcp_server_names = server_set
+        self._agent.mcp_server_mounts = server_mounts
+        self._agent.tool_whitelist = whitelist
+
+        return self
+
+    def mount(self, paths: str | list[str], *, validate: bool = False) -> AgentBuilder:
+        """Mount agent in specific directories for MCP root access.
+
+        .. deprecated:: 0.2.0
+            Use `.with_mcps({"server_name": ["/path"]})` instead for server-specific mounts.
+            This method applies mounts globally to all MCP servers.
+
+        This sets the filesystem roots that MCP servers will operate under for this agent.
+        Paths are cumulative across multiple calls.
+
+        Args:
+            paths: Single path or list of paths to mount
+            validate: If True, validate that paths exist (default: False)
+
+        Returns:
+            AgentBuilder for method chaining
+
+        Example:
+            >>> # Old way (deprecated)
+            >>> agent.with_mcps(["filesystem"]).mount("/workspace/src")
+            >>>
+            >>> # New way (recommended)
+            >>> agent.with_mcps({"filesystem": ["/workspace/src"]})
+        """
+        import warnings
+
+        warnings.warn(
+            "Agent.mount() is deprecated. Use .with_mcps({'server': ['/path']}) "
+            "for server-specific mounts instead.",
+            DeprecationWarning,
+            stacklevel=2,
+        )
+
+        if isinstance(paths, str):
+            paths = [paths]
+        if validate:
+            from pathlib import Path
+
+            for path in paths:
+                if not Path(path).exists():
+                    raise ValueError(f"Mount path does not exist: {path}")
+
+        # Add to agent's mount points (cumulative) - for backward compatibility
+        self._agent.mcp_mount_points.extend(paths)
+
+        # Also add to all configured servers for backward compatibility
+        for server_name in self._agent.mcp_server_names:
+            if server_name not in self._agent.mcp_server_mounts:
+                self._agent.mcp_server_mounts[server_name] = []
+            self._agent.mcp_server_mounts[server_name].extend(paths)
 
         return self
 

flock/cli.py

@@ -3,14 +3,17 @@
 from __future__ import annotations
 
 import asyncio
+from datetime import datetime
 
 import typer
 from rich.console import Console
 from rich.table import Table
+from typer.models import OptionInfo
 
 # Lazy import: only import examples when CLI commands are invoked
 # This prevents polluting type_registry on every package import
 from flock.service import BlackboardHTTPService
+from flock.store import SQLiteBlackboardStore
 
 
 app = typer.Typer(help="Blackboard Agents CLI")
@@ -58,16 +61,85 @@ def list_agents() -> None:
 
 
 @app.command()
-def serve(host: str = "127.0.0.1", port: int = 8000) -> None:
+def serve(
+    host: str = "127.0.0.1",
+    port: int = 8000,
+    sqlite_db: str | None = typer.Option(None, help="Path to SQLite blackboard store"),
+) -> None:
     """Run the HTTP control plane bound to the demo orchestrator."""
 
     from flock.examples import create_demo_orchestrator
 
-    orchestrator, _ = create_demo_orchestrator()
+    if isinstance(sqlite_db, OptionInfo):  # Allow direct invocation in tests
+        sqlite_db = sqlite_db.default
+
+    store = None
+    if sqlite_db is not None:
+        sqlite_store = SQLiteBlackboardStore(sqlite_db)
+
+        async def _prepare() -> SQLiteBlackboardStore:
+            await sqlite_store.ensure_schema()
+            return sqlite_store
+
+        store = asyncio.run(_prepare())
+
+    orchestrator, _ = create_demo_orchestrator(store=store)
     service = BlackboardHTTPService(orchestrator)
     service.run(host=host, port=port)
 
 
+@app.command("init-sqlite-store")
+def init_sqlite_store(
+    db_path: str = typer.Argument(..., help="Path to SQLite blackboard database"),
+) -> None:
+    """Initialise the SQLite store schema."""
+
+    store = SQLiteBlackboardStore(db_path)
+
+    async def _init() -> None:
+        await store.ensure_schema()
+        await store.close()
+
+    asyncio.run(_init())
+    console.print(f"[green]Initialised SQLite blackboard at {db_path}[/green]")
+
+
+@app.command("sqlite-maintenance")
+def sqlite_maintenance(
+    db_path: str = typer.Argument(..., help="Path to SQLite blackboard database"),
+    delete_before: str | None = typer.Option(
+        None, help="ISO timestamp; delete artifacts before this time"
+    ),
+    vacuum: bool = typer.Option(False, help="Run VACUUM after maintenance"),
+) -> None:
+    """Perform maintenance tasks for the SQLite store."""
+
+    store = SQLiteBlackboardStore(db_path)
+
+    async def _maintain() -> tuple[int, bool]:
+        await store.ensure_schema()
+        deleted = 0
+        if delete_before is not None:
+            try:
+                before_dt = datetime.fromisoformat(delete_before)
+            except ValueError as exc:  # pragma: no cover - Typer handles but defensive
+                raise typer.BadParameter(f"Invalid ISO timestamp: {delete_before}") from exc
+            deleted = await store.delete_before(before_dt)
+        if vacuum:
+            await store.vacuum()
+        await store.close()
+        return deleted, vacuum
+
+    deleted, vacuum_run = asyncio.run(_maintain())
+    console.print(
+        f"[yellow]Deleted {deleted} artifacts[/yellow]"
+        if delete_before is not None
+        else "[yellow]No deletions requested[/yellow]"
+    )
+    if vacuum_run:
+        console.print("[yellow]VACUUM completed[/yellow]")
+
+
 def main() -> None:
     app()
 

flock/dashboard/websocket.py

@@ -124,15 +124,26 @@ class WebSocketManager:
         # Broadcast to all clients concurrently
         # Use return_exceptions=True to handle client failures gracefully
         # Use send_text() for FastAPI WebSocket (send JSON string as text)
+        # CRITICAL: Add timeout to prevent deadlock when client send buffer is full
         clients_list = list(self.clients)  # Copy to avoid modification during iteration
-        send_tasks = [client.send_text(message) for client in clients_list]
+
+        send_tasks = [
+            asyncio.wait_for(client.send_text(message), timeout=0.5)  # 500ms timeout
+            for client in clients_list
+        ]
         results = await asyncio.gather(*send_tasks, return_exceptions=True)
 
         # Remove clients that failed to receive the message
         failed_clients = []
         for client, result in zip(clients_list, results, strict=False):
             if isinstance(result, Exception):
-                logger.warning(f"Failed to send to client: {result}")
+                # Check if it's a timeout (backpressure) or other error
+                if isinstance(result, asyncio.TimeoutError):
+                    logger.warning(
+                        "Client send timeout (backpressure) - client is slow or disconnected, removing client"
+                    )
+                else:
+                    logger.warning(f"Failed to send to client: {result}")
                 failed_clients.append(client)
 
         # Clean up failed clients

flock/engines/dspy_engine.py

@@ -2,6 +2,7 @@
 
 from __future__ import annotations
 
+import asyncio
 import json
 import os
 from collections import OrderedDict, defaultdict
@@ -395,10 +396,46 @@ class DSPyEngine(EngineComponent):
         if isinstance(raw, BaseModel):
             return raw.model_dump()
         if isinstance(raw, str):
-            try:
-                return json.loads(raw)
-            except json.JSONDecodeError:
-                return {"text": raw}
+            text = raw.strip()
+            candidates: list[str] = []
+
+            # Primary attempt - full string
+            if text:
+                candidates.append(text)
+
+            # Handle DSPy streaming markers like `[[ ## output ## ]]`
+            if text.startswith("[[") and "]]" in text:
+                _, remainder = text.split("]]", 1)
+                remainder = remainder.strip()
+                if remainder:
+                    candidates.append(remainder)
+
+            # Handle Markdown-style fenced blocks
+            if text.startswith("```") and text.endswith("```"):
+                fenced = text.strip("`").strip()
+                if fenced:
+                    candidates.append(fenced)
+
+            # Extract first JSON-looking segment if present
+            for opener, closer in (("{", "}"), ("[", "]")):
+                start = text.find(opener)
+                end = text.rfind(closer)
+                if start != -1 and end != -1 and end > start:
+                    segment = text[start : end + 1].strip()
+                    if segment:
+                        candidates.append(segment)
+
+            seen: set[str] = set()
+            for candidate in candidates:
+                if candidate in seen:
+                    continue
+                seen.add(candidate)
+                try:
+                    return json.loads(candidate)
+                except json.JSONDecodeError:
+                    continue
+
+            return {"text": text}
         if isinstance(raw, Mapping):
             return dict(raw)
         return {"value": raw}
@@ -562,6 +599,9 @@ class DSPyEngine(EngineComponent):
         stream_buffers[status_field] = []
         stream_sequence = 0  # Monotonic sequence for ordering
 
+        # Track background WebSocket broadcast tasks to prevent garbage collection
+        ws_broadcast_tasks: set[asyncio.Task] = set()
+
         formatter = theme_dict = styles = agent_label = None
         live_cm = nullcontext()
         overflow_mode = self.stream_vertical_overflow
@@ -607,7 +647,7 @@ class DSPyEngine(EngineComponent):
                         stream_buffers[status_field].append(str(token) + "\n")
                         display_data["status"] = "".join(stream_buffers[status_field])
 
-                        # Emit to WebSocket
+                        # Emit to WebSocket (non-blocking to prevent deadlock)
                         if ws_manager and token:
                             try:
                                 event = StreamingOutputEvent(
@@ -621,10 +661,15 @@ class DSPyEngine(EngineComponent):
                                     sequence=stream_sequence,
                                     is_final=False,
                                 )
-                                await ws_manager.broadcast(event)
+                                # Use create_task to avoid blocking the streaming loop
+                                task = asyncio.create_task(ws_manager.broadcast(event))
+                                ws_broadcast_tasks.add(task)
+                                task.add_done_callback(ws_broadcast_tasks.discard)
                                 stream_sequence += 1
                             except Exception as e:
                                 logger.warning(f"Failed to emit streaming event: {e}")
+                        else:
+                            logger.exception("NO WS_MANAGER PRESENT!!!!")
 
                         if formatter is not None:
                             _refresh_panel()
@@ -643,7 +688,7 @@ class DSPyEngine(EngineComponent):
                                 stream_buffers[buffer_key]
                             )
 
-                            # Emit to WebSocket
+                            # Emit to WebSocket (non-blocking to prevent deadlock)
                             if ws_manager:
                                 logger.info(
                                     f"[STREAMING] Emitting StreamResponse token='{token}', sequence={stream_sequence}"
@@ -660,7 +705,10 @@ class DSPyEngine(EngineComponent):
                                         sequence=stream_sequence,
                                         is_final=False,
                                     )
-                                    await ws_manager.broadcast(event)
+                                    # Use create_task to avoid blocking the streaming loop
+                                    task = asyncio.create_task(ws_manager.broadcast(event))
+                                    ws_broadcast_tasks.add(task)
+                                    task.add_done_callback(ws_broadcast_tasks.discard)
                                     stream_sequence += 1
                                 except Exception as e:
                                     logger.warning(f"Failed to emit streaming event: {e}")
@@ -690,7 +738,7 @@ class DSPyEngine(EngineComponent):
                         stream_buffers[status_field].append(str(token))
                         display_data["status"] = "".join(stream_buffers[status_field])
 
-                    # Emit to WebSocket
+                    # Emit to WebSocket (non-blocking to prevent deadlock)
                     if ws_manager and token:
                         try:
                             event = StreamingOutputEvent(
@@ -704,7 +752,10 @@ class DSPyEngine(EngineComponent):
                                 sequence=stream_sequence,
                                 is_final=False,
                             )
-                            await ws_manager.broadcast(event)
+                            # Use create_task to avoid blocking the streaming loop
+                            task = asyncio.create_task(ws_manager.broadcast(event))
+                            ws_broadcast_tasks.add(task)
+                            task.add_done_callback(ws_broadcast_tasks.discard)
                             stream_sequence += 1
                         except Exception as e:
                             logger.warning(f"Failed to emit streaming event: {e}")
@@ -716,7 +767,7 @@ class DSPyEngine(EngineComponent):
                 if isinstance(value, dspy_mod.Prediction):
                     final_result = value
 
-                    # Emit final streaming event
+                    # Emit final streaming event (non-blocking to prevent deadlock)
                     if ws_manager:
                         try:
                             event = StreamingOutputEvent(
@@ -730,7 +781,10 @@ class DSPyEngine(EngineComponent):
                                 sequence=stream_sequence,
                                 is_final=True,  # Mark as final
                             )
-                            await ws_manager.broadcast(event)
+                            # Use create_task to avoid blocking the streaming loop
+                            task = asyncio.create_task(ws_manager.broadcast(event))
+                            ws_broadcast_tasks.add(task)
+                            task.add_done_callback(ws_broadcast_tasks.discard)
                             event = StreamingOutputEvent(
                                 correlation_id=str(ctx.correlation_id)
                                 if ctx and ctx.correlation_id
@@ -742,7 +796,10 @@ class DSPyEngine(EngineComponent):
                                 sequence=stream_sequence,
                                 is_final=True,  # Mark as final
                             )
-                            await ws_manager.broadcast(event)
+                            # Use create_task to avoid blocking the streaming loop
+                            task = asyncio.create_task(ws_manager.broadcast(event))
+                            ws_broadcast_tasks.add(task)
+                            task.add_done_callback(ws_broadcast_tasks.discard)
                         except Exception as e:
                             logger.warning(f"Failed to emit final streaming event: {e}")
 

flock/examples.py

@@ -18,6 +18,7 @@ from flock.components import EngineComponent
 from flock.orchestrator import Flock
 from flock.registry import flock_tool, flock_type, type_registry
 from flock.runtime import EvalInputs, EvalResult
+from flock.store import BlackboardStore
 from flock.utilities import LoggingUtility, MetricsUtility
 
 
@@ -75,8 +76,10 @@ class TaglineEngine(EngineComponent):
 
 def create_demo_orchestrator(
     model: str | None = None,
+    *,
+    store: BlackboardStore | None = None,
 ) -> tuple[Flock, dict[str, AgentBuilder]]:
-    orchestrator = Flock(model)
+    orchestrator = Flock(model, store=store)
 
     movie = (
         orchestrator.agent("movie")

flock/frontend/README.md

@@ -33,7 +33,7 @@ The dashboard offers two complementary visualization modes:
 
 ### Extensible Module System
 - **Custom Visualizations**: Add specialized views via the module system
-- **Event Log Module**: Built-in table view for detailed event inspection
+- **Historical Blackboard Module**: Persisted artifact browser with retention insights
 - **Trace Viewer Module**: Jaeger-style distributed tracing with timeline and statistics
 - **Context Menu Integration**: Right-click to add modules at any location
 - **Persistent Layout**: Module positions and sizes are saved across sessions
@@ -123,6 +123,20 @@ Every traced operation captures:
 - **Multi-Trace Comparison**: Open related traces to compare execution patterns
 - **JSON Navigation**: Use "Expand All" for complex nested structures
 
+### Historical Blackboard Module 📚
+
+The new Historical Blackboard module brings persisted artifacts into the dashboard so operators can rewind the blackboard, not just watch the live firehose.
+
+#### Highlights
+
+- **SQLite-first loading**: Fetches paginated artifacts before WebSocket replay, so the graph and detail views start with real history.
+- **Rich filtering**: Mirrors server-side `FilterConfig` capabilities (type, producer, tags, visibility, correlation, time range) with multi-select controls and saved presets.
+- **Consumption awareness**: Displays who consumed each artifact, run IDs, and consumption timestamps—ideal for reconciling downstream behaviour.
+- **Retention transparency**: Inline banners show the oldest/latest artifacts on disk and whether additional data can be loaded.
+- **Virtualized table**: Efficiently scroll through thousands of artifacts with keyboard navigation, quick selection, and payload inspection via the JSON renderer.
+
+Launch the module via the context menu (or `Add Module → Historical Blackboard`) after running `examples/03-the-dashboard/04_persistent_pizza_dashboard.py` against a SQLite-backed orchestrator.
+
 ### Modern UI/UX
 - **Glassmorphism Design**: Modern dark theme with semi-transparent surfaces and blur effects
 - **Keyboard Shortcuts**: Navigate efficiently with Ctrl+M, Ctrl+F, and Esc