flock-core 0.5.0b65__py3-none-any.whl → 0.5.0b71__py3-none-any.whl

flock/cli.py

@@ -3,14 +3,17 @@
 from __future__ import annotations
 
 import asyncio
+from datetime import datetime
 
 import typer
 from rich.console import Console
 from rich.table import Table
+from typer.models import OptionInfo
 
 # Lazy import: only import examples when CLI commands are invoked
 # This prevents polluting type_registry on every package import
 from flock.service import BlackboardHTTPService
+from flock.store import SQLiteBlackboardStore
 
 
 app = typer.Typer(help="Blackboard Agents CLI")
@@ -58,16 +61,85 @@ def list_agents() -> None:
 
 
 @app.command()
-def serve(host: str = "127.0.0.1", port: int = 8000) -> None:
+def serve(
+    host: str = "127.0.0.1",
+    port: int = 8000,
+    sqlite_db: str | None = typer.Option(None, help="Path to SQLite blackboard store"),
+) -> None:
     """Run the HTTP control plane bound to the demo orchestrator."""
 
     from flock.examples import create_demo_orchestrator
 
-    orchestrator, _ = create_demo_orchestrator()
+    if isinstance(sqlite_db, OptionInfo):  # Allow direct invocation in tests
+        sqlite_db = sqlite_db.default
+
+    store = None
+    if sqlite_db is not None:
+        sqlite_store = SQLiteBlackboardStore(sqlite_db)
+
+        async def _prepare() -> SQLiteBlackboardStore:
+            await sqlite_store.ensure_schema()
+            return sqlite_store
+
+        store = asyncio.run(_prepare())
+
+    orchestrator, _ = create_demo_orchestrator(store=store)
     service = BlackboardHTTPService(orchestrator)
     service.run(host=host, port=port)
 
 
+@app.command("init-sqlite-store")
+def init_sqlite_store(
+    db_path: str = typer.Argument(..., help="Path to SQLite blackboard database"),
+) -> None:
+    """Initialise the SQLite store schema."""
+
+    store = SQLiteBlackboardStore(db_path)
+
+    async def _init() -> None:
+        await store.ensure_schema()
+        await store.close()
+
+    asyncio.run(_init())
+    console.print(f"[green]Initialised SQLite blackboard at {db_path}[/green]")
+
+
+@app.command("sqlite-maintenance")
+def sqlite_maintenance(
+    db_path: str = typer.Argument(..., help="Path to SQLite blackboard database"),
+    delete_before: str | None = typer.Option(
+        None, help="ISO timestamp; delete artifacts before this time"
+    ),
+    vacuum: bool = typer.Option(False, help="Run VACUUM after maintenance"),
+) -> None:
+    """Perform maintenance tasks for the SQLite store."""
+
+    store = SQLiteBlackboardStore(db_path)
+
+    async def _maintain() -> tuple[int, bool]:
+        await store.ensure_schema()
+        deleted = 0
+        if delete_before is not None:
+            try:
+                before_dt = datetime.fromisoformat(delete_before)
+            except ValueError as exc:  # pragma: no cover - Typer handles but defensive
+                raise typer.BadParameter(f"Invalid ISO timestamp: {delete_before}") from exc
+            deleted = await store.delete_before(before_dt)
+        if vacuum:
+            await store.vacuum()
+        await store.close()
+        return deleted, vacuum
+
+    deleted, vacuum_run = asyncio.run(_maintain())
+    console.print(
+        f"[yellow]Deleted {deleted} artifacts[/yellow]"
+        if delete_before is not None
+        else "[yellow]No deletions requested[/yellow]"
+    )
+    if vacuum_run:
+        console.print("[yellow]VACUUM completed[/yellow]")
+
+
 def main() -> None:
     app()
 

flock/engines/dspy_engine.py

@@ -396,10 +396,46 @@ class DSPyEngine(EngineComponent):
         if isinstance(raw, BaseModel):
             return raw.model_dump()
         if isinstance(raw, str):
-            try:
-                return json.loads(raw)
-            except json.JSONDecodeError:
-                return {"text": raw}
+            text = raw.strip()
+            candidates: list[str] = []
+
+            # Primary attempt - full string
+            if text:
+                candidates.append(text)
+
+            # Handle DSPy streaming markers like `[[ ## output ## ]]`
+            if text.startswith("[[") and "]]" in text:
+                _, remainder = text.split("]]", 1)
+                remainder = remainder.strip()
+                if remainder:
+                    candidates.append(remainder)
+
+            # Handle Markdown-style fenced blocks
+            if text.startswith("```") and text.endswith("```"):
+                fenced = text.strip("`").strip()
+                if fenced:
+                    candidates.append(fenced)
+
+            # Extract first JSON-looking segment if present
+            for opener, closer in (("{", "}"), ("[", "]")):
+                start = text.find(opener)
+                end = text.rfind(closer)
+                if start != -1 and end != -1 and end > start:
+                    segment = text[start : end + 1].strip()
+                    if segment:
+                        candidates.append(segment)
+
+            seen: set[str] = set()
+            for candidate in candidates:
+                if candidate in seen:
+                    continue
+                seen.add(candidate)
+                try:
+                    return json.loads(candidate)
+                except json.JSONDecodeError:
+                    continue
+
+            return {"text": text}
         if isinstance(raw, Mapping):
             return dict(raw)
         return {"value": raw}
@@ -633,7 +669,7 @@ class DSPyEngine(EngineComponent):
                             except Exception as e:
                                 logger.warning(f"Failed to emit streaming event: {e}")
                         else:
-                            logger.exception("NO WS_MANAGER PRESENT!!!!")
+                            logger.debug("No WebSocket manager present for streaming event.")
 
                         if formatter is not None:
                             _refresh_panel()

flock/examples.py

@@ -18,6 +18,7 @@ from flock.components import EngineComponent
 from flock.orchestrator import Flock
 from flock.registry import flock_tool, flock_type, type_registry
 from flock.runtime import EvalInputs, EvalResult
+from flock.store import BlackboardStore
 from flock.utilities import LoggingUtility, MetricsUtility
 
 
@@ -75,8 +76,10 @@ class TaglineEngine(EngineComponent):
 
 def create_demo_orchestrator(
     model: str | None = None,
+    *,
+    store: BlackboardStore | None = None,
 ) -> tuple[Flock, dict[str, AgentBuilder]]:
-    orchestrator = Flock(model)
+    orchestrator = Flock(model, store=store)
 
     movie = (
         orchestrator.agent("movie")

flock/frontend/README.md

@@ -33,7 +33,7 @@ The dashboard offers two complementary visualization modes:
 
 ### Extensible Module System
 - **Custom Visualizations**: Add specialized views via the module system
-- **Event Log Module**: Built-in table view for detailed event inspection
+- **Historical Blackboard Module**: Persisted artifact browser with retention insights
 - **Trace Viewer Module**: Jaeger-style distributed tracing with timeline and statistics
 - **Context Menu Integration**: Right-click to add modules at any location
 - **Persistent Layout**: Module positions and sizes are saved across sessions
@@ -123,6 +123,20 @@ Every traced operation captures:
 - **Multi-Trace Comparison**: Open related traces to compare execution patterns
 - **JSON Navigation**: Use "Expand All" for complex nested structures
 
+### Historical Blackboard Module 📚
+
+The new Historical Blackboard module brings persisted artifacts into the dashboard so operators can rewind the blackboard, not just watch the live firehose.
+
+#### Highlights
+
+- **SQLite-first loading**: Fetches paginated artifacts before WebSocket replay, so the graph and detail views start with real history.
+- **Rich filtering**: Mirrors server-side `FilterConfig` capabilities (type, producer, tags, visibility, correlation, time range) with multi-select controls and saved presets.
+- **Consumption awareness**: Displays who consumed each artifact, run IDs, and consumption timestamps—ideal for reconciling downstream behaviour.
+- **Retention transparency**: Inline banners show the oldest/latest artifacts on disk and whether additional data can be loaded.
+- **Virtualized table**: Efficiently scroll through thousands of artifacts with keyboard navigation, quick selection, and payload inspection via the JSON renderer.
+
+Launch the module via the context menu (or `Add Module → Historical Blackboard`) after running `examples/03-the-dashboard/04_persistent_pizza_dashboard.py` against a SQLite-backed orchestrator.
+
 ### Modern UI/UX
 - **Glassmorphism Design**: Modern dark theme with semi-transparent surfaces and blur effects
 - **Keyboard Shortcuts**: Navigate efficiently with Ctrl+M, Ctrl+F, and Esc

flock/frontend/package-lock.json

@@ -1,12 +1,12 @@
 {
   "name": "flock-ui",
-  "version": "0.1.4",
+  "version": "0.1.7",
   "lockfileVersion": 3,
   "requires": true,
   "packages": {
     "": {
       "name": "flock-ui",
-      "version": "0.1.4",
+      "version": "0.1.7",
       "license": "ISC",
       "dependencies": {
         "@types/dagre": "^0.7.53",

flock/frontend/package.json

@@ -1,6 +1,6 @@
 {
   "name": "flock-ui",
-  "version": "0.1.4",
+  "version": "0.1.7",
   "description": "Flock Flow Real-Time Dashboard Frontend",
   "type": "module",
   "scripts": {

flock/frontend/src/App.tsx

@@ -5,9 +5,12 @@ import { measureRenderTime } from './utils/performance';
 import { initializeWebSocket } from './services/websocket';
 import { registerModules } from './components/modules/registerModules';
 import { useKeyboardShortcuts } from './hooks/useKeyboardShortcuts';
-import { fetchRegisteredAgents } from './services/api';
+import { fetchRegisteredAgents, fetchArtifactSummary, fetchArtifacts } from './services/api';
 import { useGraphStore } from './store/graphStore';
 import { useUIStore } from './store/uiStore';
+import { useFilterStore } from './store/filterStore';
+import { mapArtifactToMessage } from './utils/artifacts';
+import { indexedDBService } from './services/indexeddb';
 
 // Register modules once at module load time
 registerModules();
@@ -30,10 +33,59 @@ const App: React.FC = () => {
       }
     });
 
-    // Initialize WebSocket connection
-    const wsUrl = import.meta.env.VITE_WS_URL || 'ws://localhost:8000/ws';
-    const wsClient = initializeWebSocket(wsUrl);
-    wsClient.connect();
+    const loadHistoricalData = async () => {
+      try {
+        await indexedDBService.initialize();
+
+        const filterStore = useFilterStore.getState();
+        const graphStore = useGraphStore.getState();
+        const uiStore = useUIStore.getState();
+
+        const summary = await fetchArtifactSummary();
+        filterStore.setSummary(summary);
+        filterStore.updateAvailableFacets({
+          artifactTypes: Object.keys(summary.by_type),
+          producers: Object.keys(summary.by_producer),
+          tags: Object.keys(summary.tag_counts),
+          visibilities: Object.keys(summary.by_visibility),
+        });
+
+        const artifactResponse = await fetchArtifacts({ limit: 200, embedMeta: true });
+        const messages = artifactResponse.items.map(mapArtifactToMessage);
+        if (messages.length > 0) {
+          graphStore.batchUpdate({ messages });
+          if (uiStore.mode === 'blackboard') {
+            graphStore.generateBlackboardViewGraph();
+          } else {
+            graphStore.generateAgentViewGraph();
+          }
+          graphStore.applyFilters();
+
+          const correlationMetadata = new Map<string, { correlation_id: string; first_seen: number; artifact_count: number; run_count: number }>();
+          artifactResponse.items.forEach((item) => {
+            if (!item.correlation_id) return;
+            const timestamp = new Date(item.created_at).getTime();
+            const existing = correlationMetadata.get(item.correlation_id);
+            if (existing) {
+              existing.artifact_count += 1;
+              existing.first_seen = Math.min(existing.first_seen, timestamp);
+            } else {
+              correlationMetadata.set(item.correlation_id, {
+                correlation_id: item.correlation_id,
+                first_seen: timestamp,
+                artifact_count: 1,
+                run_count: 0,
+              });
+            }
+          });
+          if (correlationMetadata.size > 0) {
+            filterStore.updateAvailableCorrelationIds(Array.from(correlationMetadata.values()));
+          }
+        }
+      } catch (error) {
+        console.error('[App] Failed to load historical artifacts:', error);
+      }
+    };
 
     // Load registered agents from orchestrator
     // This pre-populates the graph with all agent nodes before any events occur
@@ -61,10 +113,26 @@ const App: React.FC = () => {
       }
     };
 
-    loadInitialAgents();
+    const wsUrl = import.meta.env.VITE_WS_URL || 'ws://localhost:8000/ws';
+    const wsClient = initializeWebSocket(wsUrl);
+    let cancelled = false;
+
+    const bootstrap = async () => {
+      await loadHistoricalData();
+      await loadInitialAgents();
+
+      if (!cancelled) {
+        wsClient.connect();
+      }
+    };
+
+    bootstrap().catch((error) => {
+      console.error('[App] Bootstrap failed:', error);
+    });
 
     // Cleanup on unmount
     return () => {
+      cancelled = true;
       wsClient.disconnect();
     };
   }, []);

flock/frontend/src/__tests__/e2e/critical-scenarios.test.tsx

@@ -404,18 +404,17 @@ describe('Critical E2E Scenarios (Frontend)', () => {
         artifact_count: 1,
         run_count: 1,
       }));
+      // Reset filters and generate graph with all events (use fresh state)
+      useFilterStore.getState().clearFilters();
       useFilterStore.getState().updateAvailableCorrelationIds(metadata);
 
-      // Generate graph with all events (use fresh state)
       await act(async () => {
         useGraphStore.getState().generateBlackboardViewGraph();
       });
 
-      // Wait for nodes to be generated
+      // Ensure messages were recorded
       await waitFor(() => {
-        const nodes = useGraphStore.getState().nodes;
-        const visibleNodes = nodes.filter((n) => !n.hidden);
-        expect(visibleNodes.length).toBe(3);
+        expect(useGraphStore.getState().messages.size).toBe(3);
       }, { timeout: 5000 });
 
       // Apply correlation ID filter (use fresh state)

flock/frontend/src/__tests__/integration/filtering-e2e.test.tsx

@@ -329,9 +329,13 @@ describe('Filtering Integration E2E', () => {
 
       const state = useGraphStore.getState();
       const visibleNodes = state.nodes.filter((n) => !n.hidden);
-      // Only agent-1 should be visible
-      expect(visibleNodes).toHaveLength(1);
-      expect(visibleNodes[0]?.id).toBe('agent-1');
+
+      // Agent nodes remain visible but metrics reflect filtered artifacts
+      expect(visibleNodes).toHaveLength(2);
+      const agent1Node = visibleNodes.find((n) => n.id === 'agent-1');
+      const agent2Node = visibleNodes.find((n) => n.id === 'agent-2');
+      expect(agent1Node).toBeDefined();
+      expect(agent2Node).toBeDefined();
     });
   });
 

flock/frontend/src/components/filters/ArtifactTypeFilter.tsx

@@ -0,0 +1,21 @@
+import React from 'react';
+import MultiSelect from '../settings/MultiSelect';
+import { useFilterStore } from '../../store/filterStore';
+
+const ArtifactTypeFilter: React.FC = () => {
+  const options = useFilterStore((state) => state.availableArtifactTypes);
+  const selected = useFilterStore((state) => state.selectedArtifactTypes);
+  const setArtifactTypes = useFilterStore((state) => state.setArtifactTypes);
+
+  return (
+    <MultiSelect
+      options={options}
+      selected={selected}
+      onChange={setArtifactTypes}
+      placeholder={options.length ? 'Select types…' : 'No types available'}
+      disabled={options.length === 0}
+    />
+  );
+};
+
+export default ArtifactTypeFilter;

flock/frontend/src/components/filters/FilterFlyout.module.css

@@ -0,0 +1,104 @@
+.panel {
+  position: fixed;
+  top: 0;
+  left: 0;
+  bottom: 0;
+  width: 420px;
+  background: var(--color-glass-bg);
+  backdrop-filter: blur(var(--blur-lg));
+  border-right: var(--border-default);
+  box-shadow: var(--shadow-2xl);
+  z-index: 1100;
+  animation: slideInLeft var(--duration-slow) var(--ease-smooth);
+  display: flex;
+  flex-direction: column;
+}
+
+.header {
+  display: flex;
+  align-items: flex-start;
+  justify-content: space-between;
+  gap: var(--space-component-sm);
+  padding: var(--space-layout-md);
+  border-bottom: var(--border-subtle);
+}
+
+.title {
+  margin: 0;
+  font-size: var(--font-size-h3);
+  color: var(--color-text-primary);
+}
+
+.subtitle {
+  margin: 0;
+  font-size: var(--font-size-body-xs);
+  color: var(--color-text-tertiary);
+}
+
+.closeButton {
+  width: 32px;
+  height: 32px;
+  border-radius: var(--radius-md);
+  border: none;
+  background: transparent;
+  color: var(--color-text-secondary);
+  font-size: 22px;
+  cursor: pointer;
+  transition: var(--transition-all);
+}
+
+.closeButton:hover {
+  background: var(--color-bg-overlay);
+  color: var(--color-text-primary);
+}
+
+.content {
+  flex: 1;
+  overflow-y: auto;
+  padding: var(--space-layout-md);
+  display: flex;
+  flex-direction: column;
+  gap: var(--space-layout-md);
+}
+
+.section {
+  display: flex;
+  flex-direction: column;
+  gap: var(--space-component-sm);
+}
+
+.sectionLabel {
+  margin: 0;
+  font-size: var(--font-size-body-xs);
+  letter-spacing: var(--letter-spacing-wide);
+  color: var(--color-text-tertiary);
+  text-transform: uppercase;
+}
+
+.separator {
+  width: 100%;
+  height: 1px;
+  background: linear-gradient(90deg, transparent, rgba(148, 163, 184, 0.25), transparent);
+  margin: var(--space-component-md) 0;
+}
+
+@keyframes slideInLeft {
+  from {
+    transform: translateX(-40px);
+    opacity: 0;
+  }
+  to {
+    transform: translateX(0);
+    opacity: 1;
+  }
+}
+
+@media (max-width: 960px) {
+  .panel {
+    width: 100%;
+  }
+
+  .panel {
+    width: 100%;
+  }
+}

flock/frontend/src/components/filters/FilterFlyout.tsx

@@ -0,0 +1,80 @@
+import React from 'react';
+import CorrelationIDFilter from './CorrelationIDFilter';
+import TimeRangeFilter from './TimeRangeFilter';
+import ArtifactTypeFilter from './ArtifactTypeFilter';
+import ProducerFilter from './ProducerFilter';
+import TagFilter from './TagFilter';
+import VisibilityFilter from './VisibilityFilter';
+import SavedFiltersControl from './SavedFiltersControl';
+import styles from './FilterFlyout.module.css';
+
+interface FilterFlyoutProps {
+  onClose: () => void;
+}
+
+const FilterFlyout: React.FC<FilterFlyoutProps> = ({ onClose }) => {
+  return (
+    <aside className={styles.panel} role="dialog" aria-label="Filters">
+      <header className={styles.header}>
+        <div>
+          <h2 className={styles.title}>Filters</h2>
+          <p className={styles.subtitle}>Slice historical data without losing your place.</p>
+        </div>
+        <button type="button" className={styles.closeButton} onClick={onClose} aria-label="Close filters">
+          ×
+        </button>
+      </header>
+
+      <div className={styles.content}>
+        <section className={styles.section}>
+          <h3 className={styles.sectionLabel}>Presets</h3>
+          <SavedFiltersControl />
+        </section>
+
+        <div className={styles.separator} role="presentation" />
+
+        <section className={styles.section}>
+          <h3 className={styles.sectionLabel}>Correlation</h3>
+          <CorrelationIDFilter />
+        </section>
+
+        <div className={styles.separator} role="presentation" />
+
+        <section className={styles.section}>
+          <h3 className={styles.sectionLabel}>Time Range</h3>
+          <TimeRangeFilter />
+        </section>
+
+        <div className={styles.separator} role="presentation" />
+
+        <section className={styles.section}>
+          <h3 className={styles.sectionLabel}>Artifact Types</h3>
+          <ArtifactTypeFilter />
+        </section>
+
+        <div className={styles.separator} role="presentation" />
+
+        <section className={styles.section}>
+          <h3 className={styles.sectionLabel}>Producers</h3>
+          <ProducerFilter />
+        </section>
+
+        <div className={styles.separator} role="presentation" />
+
+        <section className={styles.section}>
+          <h3 className={styles.sectionLabel}>Tags</h3>
+          <TagFilter />
+        </section>
+
+        <div className={styles.separator} role="presentation" />
+
+        <section className={styles.section}>
+          <h3 className={styles.sectionLabel}>Visibility</h3>
+          <VisibilityFilter />
+        </section>
+      </div>
+    </aside>
+  );
+};
+
+export default FilterFlyout;