flock-core 0.5.0b63__py3-none-any.whl → 0.5.0b70__py3-none-any.whl

flock/frontend/src/store/graphStore.ts

@@ -76,14 +76,79 @@ function toDashboardState(
   consumptions: Map<string, string[]>
 ): DashboardState {
   const artifacts = new Map<string, Artifact>();
+  const syntheticRuns = new Map(runs);
+
+  const producedBuckets = new Map<string, Set<string>>();
+  const consumedBuckets = new Map<string, Set<string>>();
+
+  // Helper to build bucket keys based on agent + correlation
+  const makeBucketKey = (agentId: string, correlationId: string) =>
+    `${agentId}::${correlationId || 'uncorrelated'}`;
+
+  // Track (agent, correlation) pairs that already have explicit run data
+  const existingRunBuckets = new Set<string>();
+  runs.forEach((run) => {
+    existingRunBuckets.add(makeBucketKey(run.agent_name, run.correlation_id));
+  });
 
   messages.forEach((message) => {
     artifacts.set(message.id, messageToArtifact(message, consumptions));
+
+    if (message.producedBy) {
+      const key = makeBucketKey(message.producedBy, message.correlationId);
+      if (!producedBuckets.has(key)) {
+        producedBuckets.set(key, new Set());
+      }
+      producedBuckets.get(key)!.add(message.id);
+    }
+  });
+
+  consumptions.forEach((consumerIds, artifactId) => {
+    const message = messages.get(artifactId);
+    const correlationId = message?.correlationId ?? '';
+    consumerIds.forEach((consumerId) => {
+      const key = makeBucketKey(consumerId, correlationId);
+      if (!consumedBuckets.has(key)) {
+        consumedBuckets.set(key, new Set());
+      }
+      consumedBuckets.get(key)!.add(artifactId);
+    });
+  });
+
+  let syntheticCounter = 0;
+  consumedBuckets.forEach((consumedSet, key) => {
+    if (consumedSet.size === 0) {
+      return;
+    }
+    const producedSet = producedBuckets.get(key);
+    if (!producedSet || producedSet.size === 0) {
+      return;
+    }
+
+    if (existingRunBuckets.has(key)) {
+      return;
+    }
+
+    const [agentIdRaw, correlationPartRaw] = key.split('::');
+    const agentId = agentIdRaw || 'unknown-agent';
+    const correlationPart = correlationPartRaw || 'uncorrelated';
+    const runId = `historic_${agentId}_${correlationPart}_${syntheticCounter++}`;
+
+    if (!syntheticRuns.has(runId)) {
+      syntheticRuns.set(runId, {
+        run_id: runId,
+        agent_name: agentId,
+        correlation_id: correlationPart === 'uncorrelated' ? '' : correlationPart,
+        status: 'completed',
+        consumed_artifacts: Array.from(consumedSet),
+        produced_artifacts: Array.from(producedSet),
+      });
+    }
   });
 
   return {
     artifacts,
-    runs,
+    runs: syntheticRuns,
     consumptions, // Phase 11: Pass actual consumption data for filtered count calculation
   };
 }
@@ -251,6 +316,8 @@ export const useGraphStore = create<GraphState>()(
         const edges = deriveAgentViewEdges(dashboardState);
 
         set({ nodes, edges });
+        // Re-apply active filters so newly generated nodes respect current selections
+        useGraphStore.getState().applyFilters();
       },
 
       generateBlackboardViewGraph: () => {
@@ -289,6 +356,8 @@ export const useGraphStore = create<GraphState>()(
               timestamp: message.timestamp,
               isStreaming: message.isStreaming || false,
               streamingText: message.streamingText || '',
+              tags: message.tags || [],
+              visibilityKind: message.visibilityKind || 'Unknown',
             },
           });
         });
@@ -298,6 +367,8 @@ export const useGraphStore = create<GraphState>()(
         const edges = deriveBlackboardViewEdges(dashboardState);
 
         set({ nodes, edges });
+        // Ensure filters are reapplied after regeneration
+        useGraphStore.getState().applyFilters();
       },
 
       batchUpdate: (update) =>
@@ -312,9 +383,16 @@ export const useGraphStore = create<GraphState>()(
 
           if (update.messages) {
             const messages = new Map(state.messages);
-            update.messages.forEach((m) => messages.set(m.id, m));
+            const consumptions = new Map(state.consumptions);
+            update.messages.forEach((m) => {
+              messages.set(m.id, m);
+              if (m.consumedBy && m.consumedBy.length > 0) {
+                consumptions.set(m.id, Array.from(new Set(m.consumedBy)));
+              }
+            });
             newState.messages = messages;
             newState.events = [...update.messages, ...state.events].slice(0, 100);
+            newState.consumptions = consumptions;
           }
 
           if (update.runs) {
@@ -327,8 +405,15 @@ export const useGraphStore = create<GraphState>()(
         }),
 
       applyFilters: () => {
-        const { nodes, edges, messages } = get();
-        const { correlationId, timeRange } = useFilterStore.getState();
+        const { nodes, edges, messages, consumptions } = get();
+        const {
+          correlationId,
+          timeRange,
+          selectedArtifactTypes,
+          selectedProducers,
+          selectedTags,
+          selectedVisibility,
+        } = useFilterStore.getState();
 
         // Helper to calculate time range boundaries
         const getTimeRangeBoundaries = (): { start: number; end: number } => {
@@ -342,64 +427,118 @@ export const useGraphStore = create<GraphState>()(
           } else if (timeRange.preset === 'custom' && timeRange.start && timeRange.end) {
             return { start: timeRange.start, end: timeRange.end };
           }
-          return { start: now - 10 * 60 * 1000, end: now };
+          return { start: Number.NEGATIVE_INFINITY, end: Number.POSITIVE_INFINITY };
         };
 
         const { start: timeStart, end: timeEnd } = getTimeRangeBoundaries();
 
-        // Filter messages based on correlation ID and time range
         const visibleMessageIds = new Set<string>();
+        const producedStats = new Map<string, { total: number; byType: Record<string, number> }>();
+        const consumedStats = new Map<string, { total: number; byType: Record<string, number> }>();
+
+        const incrementStat = (
+          map: Map<string, { total: number; byType: Record<string, number> }>,
+          key: string,
+          type: string
+        ) => {
+          if (!map.has(key)) {
+            map.set(key, { total: 0, byType: {} });
+          }
+          const entry = map.get(key)!;
+          entry.total += 1;
+          entry.byType[type] = (entry.byType[type] || 0) + 1;
+        };
+
         messages.forEach((message) => {
           let visible = true;
 
-          // Apply correlation ID filter (selective)
           if (correlationId && message.correlationId !== correlationId) {
             visible = false;
           }
 
-          // Apply time range filter (in-memory)
           if (visible && (message.timestamp < timeStart || message.timestamp > timeEnd)) {
             visible = false;
           }
 
+          if (
+            visible &&
+            selectedArtifactTypes.length > 0 &&
+            !selectedArtifactTypes.includes(message.type)
+          ) {
+            visible = false;
+          }
+
+          if (
+            visible &&
+            selectedProducers.length > 0 &&
+            !selectedProducers.includes(message.producedBy)
+          ) {
+            visible = false;
+          }
+
+          if (
+            visible &&
+            selectedVisibility.length > 0 &&
+            !selectedVisibility.includes(message.visibilityKind || 'Unknown')
+          ) {
+            visible = false;
+          }
+
+          if (visible && selectedTags.length > 0) {
+            const messageTags = message.tags || [];
+            const hasAllTags = selectedTags.every((tag) => messageTags.includes(tag));
+            if (!hasAllTags) {
+              visible = false;
+            }
+          }
+
           if (visible) {
             visibleMessageIds.add(message.id);
+            incrementStat(producedStats, message.producedBy, message.type);
+
+            const consumers = consumptions.get(message.id) || [];
+            consumers.forEach((consumerId) => {
+              incrementStat(consumedStats, consumerId, message.type);
+            });
           }
         });
 
-        // Update nodes visibility
         const updatedNodes = nodes.map((node) => {
           if (node.type === 'message') {
-            // For message nodes, check if message is visible
             return {
               ...node,
               hidden: !visibleMessageIds.has(node.id),
             };
-          } else if (node.type === 'agent') {
-            // For agent nodes, show if any visible messages involve this agent
-            let hasVisibleMessages = false;
-            messages.forEach((message) => {
-              if (visibleMessageIds.has(message.id)) {
-                if (message.producedBy === node.id) {
-                  hasVisibleMessages = true;
-                }
-              }
-            });
+          }
+          if (node.type === 'agent') {
+            const produced = producedStats.get(node.id);
+            const consumed = consumedStats.get(node.id);
+            const currentData = node.data as AgentNodeData;
             return {
               ...node,
-              hidden: !hasVisibleMessages,
+              hidden: false,
+              data: {
+                ...node.data,
+                sentCount: produced?.total ?? currentData.sentCount ?? 0,
+                recvCount: consumed?.total ?? currentData.recvCount ?? 0,
+                sentByType: produced?.byType ?? currentData.sentByType ?? {},
+                receivedByType: consumed?.byType ?? currentData.receivedByType ?? {},
+              },
             };
           }
           return node;
         });
 
-        // Update edges visibility
         const updatedEdges = edges.map((edge) => {
-          // Hide edge if either source or target node is hidden
-          const sourceNode = updatedNodes.find((n) => n.id === edge.source);
-          const targetNode = updatedNodes.find((n) => n.id === edge.target);
-          const hidden = sourceNode?.hidden || targetNode?.hidden || false;
-
+          let hidden = edge.hidden ?? false;
+          const data: any = edge.data;
+          if (data && Array.isArray(data.artifactIds) && data.artifactIds.length > 0) {
+            hidden = data.artifactIds.every((artifactId: string) => !visibleMessageIds.has(artifactId));
+          } else {
+            const sourceNode = updatedNodes.find((n) => n.id === edge.source);
+            const targetNode = updatedNodes.find((n) => n.id === edge.target);
+            hidden = !!(sourceNode?.hidden || targetNode?.hidden);
+          }
           return {
             ...edge,
             hidden,

flock/frontend/src/types/filters.ts

@@ -1,4 +1,4 @@
-export type TimeRangePreset = 'last5min' | 'last10min' | 'last1hour' | 'custom';
+export type TimeRangePreset = 'last5min' | 'last10min' | 'last1hour' | 'custom' | 'all';
 
 export interface TimeRange {
   preset: TimeRangePreset;
@@ -12,3 +12,36 @@ export interface CorrelationIdMetadata {
   artifact_count: number;
   run_count: number;
 }
+
+export interface FilterFacets {
+  artifactTypes: string[];
+  producers: string[];
+  tags: string[];
+  visibilities: string[];
+}
+
+export interface ArtifactSummary {
+  total: number;
+  by_type: Record<string, number>;
+  by_producer: Record<string, number>;
+  by_visibility: Record<string, number>;
+  tag_counts: Record<string, number>;
+  earliest_created_at: string | null;
+  latest_created_at: string | null;
+}
+
+export interface FilterSnapshot {
+  correlationId: string | null;
+  timeRange: TimeRange;
+  artifactTypes: string[];
+  producers: string[];
+  tags: string[];
+  visibility: string[];
+}
+
+export interface SavedFilterMeta {
+  filter_id: string;
+  name: string;
+  created_at: number;
+  filters: FilterSnapshot;
+}

flock/frontend/src/types/graph.ts

@@ -22,8 +22,13 @@ export interface Message {
   timestamp: number;
   correlationId: string;
   producedBy: string;
+  tags?: string[];
+  visibilityKind?: string;
+  partitionKey?: string | null;
+  version?: number;
   isStreaming?: boolean; // True while streaming, false when complete
   streamingText?: string; // Accumulated streaming text (raw)
+  consumedBy?: string[];
 }
 
 export interface AgentNodeData extends Record<string, unknown> {
@@ -47,6 +52,8 @@ export interface MessageNodeData extends Record<string, unknown> {
   timestamp: number;
   isStreaming?: boolean; // True while streaming tokens
   streamingText?: string; // Raw streaming text
+  tags?: string[];
+  visibilityKind?: string;
 }
 
 export type AgentViewNode = Node<AgentNodeData, 'agent'>;

flock/frontend/src/utils/artifacts.ts

@@ -0,0 +1,24 @@
+import type { ArtifactListItem } from '../services/api';
+import type { Message } from '../types/graph';
+
+export function mapArtifactToMessage(item: ArtifactListItem): Message {
+  const consumedBy =
+    item.consumed_by ??
+    (item.consumptions ? Array.from(new Set(item.consumptions.map((record) => record.consumer))) : []);
+
+  return {
+    id: item.id,
+    type: item.type,
+    payload: item.payload,
+    timestamp: new Date(item.created_at).getTime(),
+    correlationId: item.correlation_id ?? '',
+    producedBy: item.produced_by,
+    tags: item.tags || [],
+    visibilityKind: item.visibility_kind || item.visibility?.kind || 'Unknown',
+    partitionKey: item.partition_key ?? null,
+    version: item.version ?? 1,
+    isStreaming: false,
+    streamingText: '',
+    consumedBy,
+  };
+}

flock/mcp/client.py

@@ -153,6 +153,9 @@ class FlockMCPClient(BaseModel, ABC):
     class _SessionProxy:
         def __init__(self, client: Any):
             self._client = client
+            # Check if roots are specified in the config
+            if not self.current_roots and self.config.connection_config.mount_points:
+                self.current_roots = self.config.connection_config.mount_points
 
         def __getattr__(self, name: str):
             # return an async function that auto-reconnects, then calls through.
@@ -366,11 +369,20 @@ class FlockMCPClient(BaseModel, ABC):
             return result
 
         async def _get_tools_internal() -> list[FlockMCPTool]:
-            # TODO: Crash
             response: ListToolsResult = await self.session.list_tools()
             flock_tools = []
+            tool_whitelist = self.config.feature_config.tool_whitelist
 
             for tool in response.tools:
+                # Skip tools that are not whitelisted
+                # IF a whitelist is present
+                if (
+                    tool_whitelist is not None
+                    and isinstance(tool_whitelist, list)
+                    and len(tool_whitelist) > 0
+                    and tool.name not in tool_whitelist
+                ):
+                    continue
                 converted_tool = FlockMCPTool.from_mcp_tool(
                     tool,
                     agent_id=agent_id,
@@ -422,6 +434,14 @@ class FlockMCPClient(BaseModel, ABC):
         async with self.lock:
             return self.current_roots
 
+    def _get_roots_no_lock(self) -> list[MCPRoot] | None:
+        """Get the currently set roots without acquiring a lock.
+
+        WARNING: Only use this internally when you're sure there's no race condition.
+        This is primarily for use during initialization when the lock is already held.
+        """
+        return self.current_roots
+
     async def set_roots(self, new_roots: list[MCPRoot]) -> None:
         """Set the current roots of the client.
 
@@ -599,6 +619,10 @@ class FlockMCPClient(BaseModel, ABC):
         # 2) if we already know our current roots, notify the server
         #    so that it will follow up with a ListRootsRequest
         if self.current_roots and self.config.feature_config.roots_enabled:
+            logger.debug(
+                f"Notifying server '{self.config.name}' of {len(self.current_roots)} root(s): "
+                f"{[r.uri for r in self.current_roots]}"
+            )
             await self.client_session.send_roots_list_changed()
 
         # 3) Tell the server, what logging level we would like to use

flock/mcp/config.py

@@ -271,7 +271,7 @@ class FlockMCPFeatureConfiguration(BaseModel):
         default=None,
         description="Whitelist of tool names that are enabled for this MCP server. "
         "If provided, only tools with names in this list will be available "
-        "from this server. Used in conjunction with allow_all_tools setting. "
+        "from this server."
         "Note: Agent-level tool filtering is generally preferred over "
         "server-level filtering for better granular control.",
     )
@@ -313,15 +313,6 @@ class FlockMCPConfiguration(BaseModel):
 
     name: str = Field(..., description="Name of the server the client connects to.")
 
-    allow_all_tools: bool = Field(
-        default=True,
-        description="Whether to allow usage of all tools from this MCP server. "
-        "When True (default), all tools are available unless restricted "
-        "by tool_whitelist. When False, tool access is controlled entirely "
-        "by tool_whitelist (if provided). Setting to False with no whitelist "
-        "will block all tools from this server.",
-    )
-
     connection_config: FlockMCPConnectionConfiguration = Field(
         ..., description="MCP Connection Configuration for a client."
     )

flock/mcp/manager.py

@@ -68,7 +68,9 @@ class FlockMCPClientManager:
         self._pool: dict[tuple[str, str], dict[str, FlockMCPClient]] = {}
         self._lock = asyncio.Lock()
 
-    async def get_client(self, server_name: str, agent_id: str, run_id: str) -> FlockMCPClient:
+    async def get_client(
+        self, server_name: str, agent_id: str, run_id: str, mount_points: list[str] | None = None
+    ) -> FlockMCPClient:
         """Get or create an MCP client for the given context.
 
         Architecture Decision: AD005 - Lazy Connection Establishment
@@ -105,6 +107,24 @@ class FlockMCPClientManager:
                     f"(agent={agent_id}, run={run_id})"
                 )
                 config = self._configs[server_name]
+                # MCP-ROOTS: Override mount points if provided
+                if mount_points:
+                    from flock.mcp.types import MCPRoot
+
+                    # Create MCPRoot objects from paths
+                    roots = [
+                        MCPRoot(uri=f"file://{path}", name=path.split("/")[-1])
+                        for path in mount_points
+                    ]
+                    logger.info(
+                        f"Setting {len(roots)} mount point(s) for server '{server_name}' "
+                        f"(agent={agent_id}, run={run_id}): {[r.uri for r in roots]}"
+                    )
+                    # Clone config with new mount points
+                    from copy import deepcopy
+
+                    config = deepcopy(config)
+                    config.connection_config.mount_points = roots
 
                 # Instantiate the correct concrete client class based on transport type
                 # Lazy import to avoid requiring all dependencies
@@ -145,7 +165,11 @@ class FlockMCPClientManager:
             return self._pool[key][server_name]
 
     async def get_tools_for_agent(
-        self, agent_id: str, run_id: str, server_names: set[str]
+        self,
+        agent_id: str,
+        run_id: str,
+        server_names: set[str],
+        server_mounts: dict[str, list[str]] | None = None,
     ) -> dict[str, Any]:
         """Get all tools from specified servers for an agent.
 
@@ -156,6 +180,7 @@ class FlockMCPClientManager:
             agent_id: Agent requesting tools
             run_id: Current run identifier
             server_names: Set of MCP server names to fetch tools from
+            server_mounts: Optional dict mapping server names to mount points
 
         Returns:
             Dictionary mapping namespaced tool names to tool definitions
@@ -166,10 +191,16 @@ class FlockMCPClientManager:
             other servers rather than failing the entire operation.
         """
         tools = {}
+        server_mounts = server_mounts or {}
 
         for server_name in server_names:
             try:
-                client = await self.get_client(server_name, agent_id, run_id)
+                # Get mount points specific to this server
+                mount_points = server_mounts.get(server_name)
+
+                client = await self.get_client(
+                    server_name, agent_id, run_id, mount_points=mount_points
+                )
                 server_tools = await client.get_tools(agent_id, run_id)
 
                 # Apply namespacing: AD003

flock/mcp/types/callbacks.py

@@ -64,7 +64,10 @@ async def default_list_roots_callback(
 ) -> ListRootsResult | ErrorData:
     """Default List Roots Callback."""
     if associated_client.config.feature_config.roots_enabled:
-        current_roots = await associated_client.get_roots()
+        # Use lock-free version to avoid deadlock during initialization
+        # when the lock is already held by _connect()
+        current_roots = associated_client._get_roots_no_lock()
+        logger.debug(f"Server requested list/roots. Sending: {current_roots}")
         return ListRootsResult(roots=current_roots)
     return ErrorData(code=INVALID_REQUEST, message="List roots not supported.")
 

flock/orchestrator.py

@@ -3,11 +3,13 @@
 from __future__ import annotations
 
 import asyncio
+import logging
 import os
 from asyncio import Task
 from collections.abc import Iterable, Mapping, Sequence
 from contextlib import asynccontextmanager
-from typing import TYPE_CHECKING, Any
+from datetime import datetime, timezone
+from typing import TYPE_CHECKING, Any, AsyncGenerator
 from uuid import uuid4
 
 from opentelemetry import trace
@@ -27,7 +29,7 @@ from flock.mcp import (
 )
 from flock.registry import type_registry
 from flock.runtime import Context
-from flock.store import BlackboardStore, InMemoryBlackboardStore
+from flock.store import BlackboardStore, ConsumptionRecord, InMemoryBlackboardStore
 from flock.visibility import AgentIdentity, PublicVisibility, Visibility
 
 
@@ -110,6 +112,7 @@ class Flock(metaclass=AutoTracedMeta):
             ... )
         """
         self._patch_litellm_proxy_imports()
+        self._logger = logging.getLogger(__name__)
         self.model = model
         self.store: BlackboardStore = store or InMemoryBlackboardStore()
         self._agents: dict[str, Agent] = {}
@@ -193,8 +196,8 @@ class Flock(metaclass=AutoTracedMeta):
         enable_prompts_feature: bool = True,
         enable_sampling_feature: bool = True,
         enable_roots_feature: bool = True,
+        mount_points: list[str] | None = None,
         tool_whitelist: list[str] | None = None,
-        allow_all_tools: bool = True,
         read_timeout_seconds: float = 300,
         max_retries: int = 3,
         **kwargs,
@@ -212,7 +215,6 @@ class Flock(metaclass=AutoTracedMeta):
             enable_sampling_feature: Enable LLM sampling requests
             enable_roots_feature: Enable filesystem roots
             tool_whitelist: Optional list of tool names to allow
-            allow_all_tools: If True, allow all tools (subject to whitelist)
             read_timeout_seconds: Timeout for server communications
             max_retries: Connection retry attempts
 
@@ -244,12 +246,43 @@ class Flock(metaclass=AutoTracedMeta):
         else:
             transport_type = "custom"
 
+        mcp_roots = None
+        if mount_points:
+            from pathlib import Path as PathLib
+
+            from flock.mcp.types import MCPRoot
+
+            mcp_roots = []
+            for path in mount_points:
+                # Normalize the path
+                if path.startswith("file://"):
+                    # Already a file URI
+                    uri = path
+                    # Extract path from URI for name
+                    path_str = path.replace("file://", "")
+                # the test:// path-prefix is used by testing servers such as the mcp-everything server.
+                elif path.startswith("test://"):
+                    # Already a test URI
+                    uri = path
+                    # Extract path from URI for name
+                    path_str = path.replace("test://", "")
+                else:
+                    # Convert to absolute path and create URI
+                    abs_path = PathLib(path).resolve()
+                    uri = f"file://{abs_path}"
+                    path_str = str(abs_path)
+
+                # Extract a meaningful name (last component of path)
+                name = PathLib(path_str).name or path_str.rstrip("/").split("/")[-1] or "root"
+                mcp_roots.append(MCPRoot(uri=uri, name=name))
+
         # Build configuration
         connection_config = FlockMCPConnectionConfiguration(
             max_retries=max_retries,
             connection_parameters=connection_params,
             transport_type=transport_type,
             read_timeout_seconds=read_timeout_seconds,
+            mount_points=mcp_roots,
         )
 
         feature_config = FlockMCPFeatureConfiguration(
@@ -262,7 +295,6 @@ class Flock(metaclass=AutoTracedMeta):
 
         mcp_config = FlockMCPConfiguration(
             name=name,
-            allow_all_tools=allow_all_tools,
             connection_config=connection_config,
             feature_config=feature_config,
         )
@@ -862,6 +894,25 @@ class Flock(metaclass=AutoTracedMeta):
         self._record_agent_run(agent)
         await agent.execute(ctx, artifacts)
 
+        if artifacts:
+            try:
+                timestamp = datetime.now(timezone.utc)
+                records = [
+                    ConsumptionRecord(
+                        artifact_id=artifact.id,
+                        consumer=agent.name,
+                        run_id=ctx.task_id,
+                        correlation_id=str(correlation_id) if correlation_id else None,
+                        consumed_at=timestamp,
+                    )
+                    for artifact in artifacts
+                ]
+                await self.store.record_consumptions(records)
+            except NotImplementedError:
+                pass
+            except Exception as exc:  # pragma: no cover - defensive logging
+                self._logger.exception("Failed to record artifact consumption: %s", exc)
+
     # Helpers --------------------------------------------------------------
 
     def _normalize_input(