flock-core 0.5.11__py3-none-any.whl → 0.5.21__py3-none-any.whl

flock/orchestrator/server_manager.py

@@ -0,0 +1,234 @@
+"""HTTP server management for orchestrator.
+
+Handles service startup with optional dashboard integration.
+Extracted from orchestrator.py to reduce complexity.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from asyncio import Task
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+
+if TYPE_CHECKING:
+    from flock.core.orchestrator import Flock
+
+
+class ServerManager:
+    """Manages HTTP service startup for the orchestrator.
+
+    Handles both standard API mode and dashboard mode with WebSocket support.
+    """
+
+    @staticmethod
+    async def serve(
+        orchestrator: Flock,
+        *,
+        dashboard: bool = False,
+        dashboard_v2: bool = False,
+        host: str = "127.0.0.1",
+        port: int = 8344,
+        blocking: bool = True,
+    ) -> Task[None] | None:
+        """Start HTTP service for the orchestrator.
+
+        Args:
+            orchestrator: The Flock orchestrator instance to serve
+            dashboard: Enable real-time dashboard with WebSocket support (default: False)
+            dashboard_v2: Launch the new dashboard v2 frontend (implies dashboard=True)
+            host: Host to bind to (default: "127.0.0.1")
+            port: Port to bind to (default: 8344)
+            blocking: If True, blocks until server stops. If False, starts server
+                in background and returns task handle (default: True)
+
+        Returns:
+            None if blocking=True, or Task handle if blocking=False
+
+        Examples:
+            # Basic HTTP API (no dashboard) - runs until interrupted
+            await ServerManager.serve(orchestrator)
+
+            # With dashboard (WebSocket + browser launch) - runs until interrupted
+            await ServerManager.serve(orchestrator, dashboard=True)
+
+            # Non-blocking mode - start server in background
+            task = await ServerManager.serve(orchestrator, dashboard=True, blocking=False)
+            # Now you can publish messages and run other logic
+            await orchestrator.publish(my_message)
+            await orchestrator.run_until_idle()
+        """
+        # If non-blocking, start server in background task
+        if not blocking:
+            server_task = asyncio.create_task(
+                ServerManager._serve_impl(
+                    orchestrator,
+                    dashboard=dashboard,
+                    dashboard_v2=dashboard_v2,
+                    host=host,
+                    port=port,
+                )
+            )
+            # Add cleanup callback
+            server_task.add_done_callback(
+                lambda task: ServerManager._cleanup_server_callback(orchestrator, task)
+            )
+            # Store task reference for later cancellation
+            orchestrator._server_task = server_task
+            # Give server a moment to start
+            await asyncio.sleep(0.1)
+            return server_task
+
+        # Blocking mode - run server directly with cleanup
+        try:
+            await ServerManager._serve_impl(
+                orchestrator,
+                dashboard=dashboard,
+                dashboard_v2=dashboard_v2,
+                host=host,
+                port=port,
+            )
+        finally:
+            # In blocking mode, manually cleanup dashboard launcher
+            if (
+                hasattr(orchestrator, "_dashboard_launcher")
+                and orchestrator._dashboard_launcher is not None
+            ):
+                orchestrator._dashboard_launcher.stop()
+                orchestrator._dashboard_launcher = None
+        return None
+
+    @staticmethod
+    def _cleanup_server_callback(orchestrator: Flock, task: Task[None]) -> None:
+        """Cleanup callback when background server task completes."""
+        # Stop dashboard launcher if it was started
+        if (
+            hasattr(orchestrator, "_dashboard_launcher")
+            and orchestrator._dashboard_launcher is not None
+        ):
+            try:
+                orchestrator._dashboard_launcher.stop()
+            except Exception as e:
+                orchestrator._logger.warning(f"Failed to stop dashboard launcher: {e}")
+            finally:
+                orchestrator._dashboard_launcher = None
+
+        # Clear server task reference
+        if hasattr(orchestrator, "_server_task"):
+            orchestrator._server_task = None
+
+        # Log any exceptions from the task
+        try:
+            exc = task.exception()
+            if exc and not isinstance(exc, asyncio.CancelledError):
+                orchestrator._logger.error(f"Server task failed: {exc}", exc_info=exc)
+        except asyncio.CancelledError:
+            pass  # Normal cancellation
+
+    @staticmethod
+    async def _serve_impl(
+        orchestrator: Flock,
+        *,
+        dashboard: bool = False,
+        dashboard_v2: bool = False,
+        host: str = "127.0.0.1",
+        port: int = 8344,
+    ) -> None:
+        """Internal implementation of serve() - actual server logic."""
+        if dashboard_v2:
+            dashboard = True
+
+        if not dashboard:
+            # Standard service without dashboard
+            await ServerManager._serve_standard(orchestrator, host=host, port=port)
+            return
+
+        # Dashboard mode with WebSocket and event collection
+        await ServerManager._serve_dashboard(
+            orchestrator, dashboard_v2=dashboard_v2, host=host, port=port
+        )
+
+    @staticmethod
+    async def _serve_standard(orchestrator: Flock, *, host: str, port: int) -> None:
+        """Serve standard HTTP API without dashboard.
+
+        Args:
+            orchestrator: The Flock orchestrator instance
+            host: Host to bind to
+            port: Port to bind to
+        """
+        from flock.api.service import BlackboardHTTPService
+
+        service = BlackboardHTTPService(orchestrator)
+        await service.run_async(host=host, port=port)
+
+    @staticmethod
+    async def _serve_dashboard(
+        orchestrator: Flock, *, dashboard_v2: bool, host: str, port: int
+    ) -> None:
+        """Serve HTTP API with dashboard and WebSocket support.
+
+        Args:
+            orchestrator: The Flock orchestrator instance
+            dashboard_v2: Whether to use v2 dashboard frontend
+            host: Host to bind to
+            port: Port to bind to
+        """
+        from flock.core import Agent
+        from flock.dashboard.collector import DashboardEventCollector
+        from flock.dashboard.launcher import DashboardLauncher
+        from flock.dashboard.service import DashboardHTTPService
+        from flock.dashboard.websocket import WebSocketManager
+
+        # Create dashboard components
+        websocket_manager = WebSocketManager()
+        event_collector = DashboardEventCollector(store=orchestrator.store)
+        event_collector.set_websocket_manager(websocket_manager)
+        await event_collector.load_persistent_snapshots()
+
+        # Store collector reference for agents added later
+        orchestrator._dashboard_collector = event_collector
+        # Store websocket manager for real-time event emission (Phase 1.2)
+        orchestrator._websocket_manager = websocket_manager
+        # Phase 5A: Set websocket manager on EventEmitter for dashboard updates
+        orchestrator._event_emitter.set_websocket_manager(websocket_manager)
+
+        # Phase 6+7: Set class-level WebSocket broadcast wrapper (dashboard mode)
+        async def _broadcast_wrapper(event):
+            """Isolated broadcast wrapper - no reference chain to orchestrator."""
+            return await websocket_manager.broadcast(event)
+
+        Agent._websocket_broadcast_global = _broadcast_wrapper
+
+        # Inject event collector into all existing agents
+        for agent in orchestrator._agents.values():
+            # Add dashboard collector with priority ordering handled by agent
+            agent._add_utilities([event_collector])
+
+        # Start dashboard launcher (npm process + browser)
+        launcher_kwargs: dict[str, Any] = {"port": port}
+        if dashboard_v2:
+            dashboard_pkg_dir = Path(__file__).parent.parent / "dashboard"
+            launcher_kwargs["frontend_dir"] = dashboard_pkg_dir.parent / "frontend_v2"
+            launcher_kwargs["static_dir"] = dashboard_pkg_dir / "static_v2"
+
+        launcher = DashboardLauncher(**launcher_kwargs)
+        launcher.start()
+
+        # Create dashboard HTTP service
+        service = DashboardHTTPService(
+            orchestrator=orchestrator,
+            websocket_manager=websocket_manager,
+            event_collector=event_collector,
+            use_v2=dashboard_v2,
+        )
+
+        # Store launcher for cleanup
+        orchestrator._dashboard_launcher = launcher
+
+        # Run service (blocking call)
+        # Note: Cleanup is NOT done here - it's handled by:
+        # - ServerManager.serve() finally block (blocking mode)
+        # - ServerManager._cleanup_server_callback() (non-blocking mode)
+        await service.run_async(host=host, port=port)

flock/orchestrator/tracing.py

@@ -0,0 +1,147 @@
+"""Unified tracing utilities for orchestrator workflows.
+
+Handles OpenTelemetry workflow spans and trace database management.
+Extracted from orchestrator.py to reduce complexity.
+"""
+
+from __future__ import annotations
+
+from contextlib import asynccontextmanager
+from pathlib import Path
+from typing import TYPE_CHECKING, Any
+
+from opentelemetry import trace
+from opentelemetry.trace import Status, StatusCode
+
+
+if TYPE_CHECKING:
+    from collections.abc import AsyncGenerator
+
+
+class TracingManager:
+    """Manages unified tracing for orchestrator workflows.
+
+    Provides workflow span creation and trace database cleanup utilities.
+    """
+
+    def __init__(self) -> None:
+        """Initialize tracing manager."""
+        self._workflow_span = None
+
+    @asynccontextmanager
+    async def traced_run(
+        self, name: str = "workflow", flock_id: str | None = None
+    ) -> AsyncGenerator[Any, None]:
+        """Context manager for wrapping an entire execution in a single unified trace.
+
+        This creates a parent span that encompasses all operations (publish, run_until_idle, etc.)
+        within the context, ensuring they all belong to the same trace_id for better observability.
+
+        Args:
+            name: Name for the workflow trace (default: "workflow")
+            flock_id: Optional Flock instance ID for attribution
+
+        Yields:
+            The workflow span for optional manual attribute setting
+
+        Examples:
+            # Explicit workflow tracing (recommended)
+            async with tracing_manager.traced_run("pizza_workflow"):
+                await flock.publish(pizza_idea)
+                await flock.run_until_idle()
+                # All operations now share the same trace_id!
+
+            # Custom attributes
+            async with tracing_manager.traced_run("data_pipeline") as span:
+                span.set_attribute("pipeline.version", "2.0")
+                await flock.publish(data)
+                await flock.run_until_idle()
+        """
+        tracer = trace.get_tracer(__name__)
+        with tracer.start_as_current_span(name) as span:
+            # Set workflow-level attributes
+            span.set_attribute("flock.workflow", True)
+            span.set_attribute("workflow.name", name)
+            if flock_id:
+                span.set_attribute("workflow.flock_id", flock_id)
+
+            # Store span for nested operations to use
+            prev_workflow_span = self._workflow_span
+            self._workflow_span = span
+
+            try:
+                yield span
+                span.set_status(Status(StatusCode.OK))
+            except Exception as e:
+                span.set_status(Status(StatusCode.ERROR, str(e)))
+                span.record_exception(e)
+                raise
+            finally:
+                # Restore previous workflow span
+                self._workflow_span = prev_workflow_span
+
+    @property
+    def current_workflow_span(self) -> Any:
+        """Get the current workflow span (for nested operations)."""
+        return self._workflow_span
+
+    @staticmethod
+    def clear_traces(db_path: str = ".flock/traces.duckdb") -> dict[str, Any]:
+        """Clear all traces from the DuckDB database.
+
+        Useful for resetting debug sessions or cleaning up test data.
+
+        Args:
+            db_path: Path to the DuckDB database file (default: ".flock/traces.duckdb")
+
+        Returns:
+            Dictionary with operation results:
+                - deleted_count: Number of spans deleted
+                - success: Whether operation succeeded
+                - error: Error message if failed
+
+        Examples:
+            # Clear all traces
+            result = TracingManager.clear_traces()
+            print(f"Deleted {result['deleted_count']} spans")
+
+            # Custom database path
+            result = TracingManager.clear_traces(".flock/custom_traces.duckdb")
+
+            # Check if operation succeeded
+            if result['success']:
+                print("Traces cleared successfully!")
+            else:
+                print(f"Error: {result['error']}")
+        """
+        try:
+            import duckdb
+
+            db_file = Path(db_path)
+            if not db_file.exists():
+                return {
+                    "success": False,
+                    "deleted_count": 0,
+                    "error": f"Database file not found: {db_path}",
+                }
+
+            # Connect and clear
+            conn = duckdb.connect(str(db_file))
+            try:
+                # Get count before deletion
+                count_result = conn.execute("SELECT COUNT(*) FROM spans").fetchone()
+                deleted_count = count_result[0] if count_result else 0
+
+                # Delete all spans
+                conn.execute("DELETE FROM spans")
+
+                # Vacuum to reclaim space
+                conn.execute("VACUUM")
+
+                return {"success": True, "deleted_count": deleted_count, "error": None}
+
+            finally:
+                conn.close()
+
+        except Exception as e:
+            return {"success": False, "deleted_count": 0, "error": str(e)}

flock/storage/__init__.py

@@ -0,0 +1,10 @@
+"""Storage backends for Flock blackboard."""
+
+from flock.storage.sqlite.query_builder import SQLiteQueryBuilder
+from flock.storage.sqlite.schema_manager import SQLiteSchemaManager
+
+
+__all__ = [
+    "SQLiteQueryBuilder",
+    "SQLiteSchemaManager",
+]

flock/storage/artifact_aggregator.py

@@ -0,0 +1,158 @@
+"""Artifact aggregation utilities for summary statistics.
+
+Handles aggregation logic for artifact collections, computing statistics
+like type distribution, producer counts, and time ranges.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Any
+
+from flock.core.artifacts import Artifact
+from flock.utils.time_utils import format_time_span
+
+
+class ArtifactAggregator:
+    """
+    Aggregates artifact statistics for summary reports.
+
+    Provides clean separation of aggregation logic from storage implementations.
+    Each aggregation method is simple and focused.
+    """
+
+    def aggregate_by_type(self, artifacts: list[Artifact]) -> dict[str, int]:
+        """
+        Count artifacts by type.
+
+        Args:
+            artifacts: List of artifacts to aggregate
+
+        Returns:
+            Dict mapping type names to counts
+        """
+        by_type: dict[str, int] = {}
+        for artifact in artifacts:
+            by_type[artifact.type] = by_type.get(artifact.type, 0) + 1
+        return by_type
+
+    def aggregate_by_producer(self, artifacts: list[Artifact]) -> dict[str, int]:
+        """
+        Count artifacts by producer.
+
+        Args:
+            artifacts: List of artifacts to aggregate
+
+        Returns:
+            Dict mapping producer names to counts
+        """
+        by_producer: dict[str, int] = {}
+        for artifact in artifacts:
+            by_producer[artifact.produced_by] = (
+                by_producer.get(artifact.produced_by, 0) + 1
+            )
+        return by_producer
+
+    def aggregate_by_visibility(self, artifacts: list[Artifact]) -> dict[str, int]:
+        """
+        Count artifacts by visibility kind.
+
+        Args:
+            artifacts: List of artifacts to aggregate
+
+        Returns:
+            Dict mapping visibility kinds to counts
+        """
+        by_visibility: dict[str, int] = {}
+        for artifact in artifacts:
+            kind = getattr(artifact.visibility, "kind", "Unknown")
+            by_visibility[kind] = by_visibility.get(kind, 0) + 1
+        return by_visibility
+
+    def aggregate_tags(self, artifacts: list[Artifact]) -> dict[str, int]:
+        """
+        Count tag occurrences across artifacts.
+
+        Args:
+            artifacts: List of artifacts to aggregate
+
+        Returns:
+            Dict mapping tag names to occurrence counts
+        """
+        tag_counts: dict[str, int] = {}
+        for artifact in artifacts:
+            for tag in artifact.tags:
+                tag_counts[tag] = tag_counts.get(tag, 0) + 1
+        return tag_counts
+
+    def get_date_range(
+        self, artifacts: list[Artifact]
+    ) -> tuple[datetime | None, datetime | None]:
+        """
+        Find earliest and latest creation times.
+
+        Args:
+            artifacts: List of artifacts to analyze
+
+        Returns:
+            Tuple of (earliest, latest) datetimes, or (None, None) if empty
+        """
+        if not artifacts:
+            return None, None
+
+        earliest: datetime | None = None
+        latest: datetime | None = None
+
+        for artifact in artifacts:
+            if earliest is None or artifact.created_at < earliest:
+                earliest = artifact.created_at
+            if latest is None or artifact.created_at > latest:
+                latest = artifact.created_at
+
+        return earliest, latest
+
+    def build_summary(
+        self,
+        artifacts: list[Artifact],
+        total: int,
+        is_full_window: bool,
+    ) -> dict[str, Any]:
+        """
+        Build complete summary statistics for artifacts.
+
+        Args:
+            artifacts: List of artifacts to summarize
+            total: Total count (may differ from len(artifacts) if paginated)
+            is_full_window: Whether this represents all artifacts (no filters)
+
+        Returns:
+            Dictionary with complete summary statistics:
+            - total: Total artifact count
+            - by_type: Type distribution
+            - by_producer: Producer distribution
+            - by_visibility: Visibility distribution
+            - tag_counts: Tag occurrence counts
+            - earliest_created_at: ISO string of earliest artifact
+            - latest_created_at: ISO string of latest artifact
+            - is_full_window: Whether all artifacts included
+            - window_span_label: Human-readable time span
+        """
+        by_type = self.aggregate_by_type(artifacts)
+        by_producer = self.aggregate_by_producer(artifacts)
+        by_visibility = self.aggregate_by_visibility(artifacts)
+        tag_counts = self.aggregate_tags(artifacts)
+        earliest, latest = self.get_date_range(artifacts)
+
+        window_span_label = format_time_span(earliest, latest)
+
+        return {
+            "total": total,
+            "by_type": by_type,
+            "by_producer": by_producer,
+            "by_visibility": by_visibility,
+            "tag_counts": tag_counts,
+            "earliest_created_at": earliest.isoformat() if earliest else None,
+            "latest_created_at": latest.isoformat() if latest else None,
+            "is_full_window": is_full_window,
+            "window_span_label": window_span_label,
+        }

flock/storage/in_memory/__init__.py

@@ -0,0 +1,6 @@
+"""In-memory storage implementation utilities."""
+
+from __future__ import annotations
+
+
+__all__ = ["ArtifactFilter", "HistoryAggregator"]

flock/storage/in_memory/artifact_filter.py

@@ -0,0 +1,114 @@
+"""Artifact filtering utilities for in-memory storage.
+
+Provides focused filtering logic for InMemoryBlackboardStore.query_artifacts.
+Extracted from store.py to reduce complexity from B (10) to A (4).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+
+if TYPE_CHECKING:
+    from flock.core.artifacts import Artifact
+    from flock.core.store import FilterConfig
+
+
+class ArtifactFilter:
+    """
+    Filter artifacts based on FilterConfig criteria.
+
+    Separates filtering logic from query orchestration for better
+    testability and reduced complexity.
+    """
+
+    def __init__(self, filters: FilterConfig):
+        """
+        Initialize filter with configuration.
+
+        Args:
+            filters: Filter configuration with optional criteria
+        """
+        from flock.registry import type_registry
+
+        # Pre-resolve canonical types once
+        self.canonical_types: set[str] | None = None
+        if filters.type_names:
+            self.canonical_types = {
+                type_registry.resolve_name(name) for name in filters.type_names
+            }
+
+        self.produced_by = filters.produced_by or set()
+        self.correlation_id = filters.correlation_id
+        self.tags = filters.tags or set()
+        self.visibility_kinds = filters.visibility or set()
+        self.start = filters.start
+        self.end = filters.end
+
+    def matches(self, artifact: Artifact) -> bool:
+        """
+        Check if artifact matches all filter criteria.
+
+        Uses focused helper methods to keep complexity low (A-rated).
+        Each criterion is evaluated independently for clarity.
+
+        Args:
+            artifact: Artifact to check against filters
+
+        Returns:
+            True if artifact matches all criteria, False otherwise
+
+        Examples:
+            >>> filter = ArtifactFilter(FilterConfig(produced_by={"agent1"}))
+            >>> artifact = Artifact(type="Result", produced_by="agent1", ...)
+            >>> filter.matches(artifact)
+            True
+        """
+        return (
+            self._matches_type(artifact)
+            and self._matches_producer(artifact)
+            and self._matches_correlation(artifact)
+            and self._matches_tags(artifact)
+            and self._matches_visibility(artifact)
+            and self._matches_time_range(artifact)
+        )
+
+    def _matches_type(self, artifact: Artifact) -> bool:
+        """Check if artifact type matches filter."""
+        if not self.canonical_types:
+            return True
+        return artifact.type in self.canonical_types
+
+    def _matches_producer(self, artifact: Artifact) -> bool:
+        """Check if artifact producer matches filter."""
+        if not self.produced_by:
+            return True
+        return artifact.produced_by in self.produced_by
+
+    def _matches_correlation(self, artifact: Artifact) -> bool:
+        """Check if artifact correlation ID matches filter."""
+        if not self.correlation_id:
+            return True
+        if artifact.correlation_id is None:
+            return False
+        return str(artifact.correlation_id) == self.correlation_id
+
+    def _matches_tags(self, artifact: Artifact) -> bool:
+        """Check if artifact has all required tags."""
+        if not self.tags:
+            return True
+        return self.tags.issubset(artifact.tags)
+
+    def _matches_visibility(self, artifact: Artifact) -> bool:
+        """Check if artifact visibility kind matches filter."""
+        if not self.visibility_kinds:
+            return True
+        return artifact.visibility.kind in self.visibility_kinds
+
+    def _matches_time_range(self, artifact: Artifact) -> bool:
+        """Check if artifact creation time is within range."""
+        if self.start and artifact.created_at < self.start:
+            return False
+        if self.end and artifact.created_at > self.end:
+            return False
+        return True