flock-core 0.5.2__py3-none-any.whl → 0.5.4__py3-none-any.whl

flock/orchestrator.py

@@ -18,7 +18,10 @@ from opentelemetry.trace import Status, StatusCode
 from pydantic import BaseModel
 
 from flock.agent import Agent, AgentBuilder
+from flock.artifact_collector import ArtifactCollector
 from flock.artifacts import Artifact
+from flock.batch_accumulator import BatchEngine
+from flock.correlation_engine import CorrelationEngine
 from flock.helper.cli_helper import init_console
 from flock.logging.auto_trace import AutoTracedMeta
 from flock.mcp import (
@@ -128,6 +131,14 @@ class Flock(metaclass=AutoTracedMeta):
         self.max_agent_iterations: int = max_agent_iterations
         self._agent_iteration_count: dict[str, int] = {}
         self.is_dashboard: bool = False
+        # AND gate logic: Artifact collection for multi-type subscriptions
+        self._artifact_collector = ArtifactCollector()
+        # JoinSpec logic: Correlation engine for correlated AND gates
+        self._correlation_engine = CorrelationEngine()
+        # BatchSpec logic: Batch accumulator for size/timeout batching
+        self._batch_engine = BatchEngine()
+        # Phase 1.2: WebSocket manager for real-time dashboard events (set by serve())
+        self._websocket_manager: Any = None
         # Unified tracing support
         self._workflow_span = None
         self._auto_workflow_enabled = os.getenv("FLOCK_AUTO_WORKFLOW_TRACE", "false").lower() in {
@@ -593,6 +604,8 @@ class Flock(metaclass=AutoTracedMeta):
 
         # Store collector reference for agents added later
         self._dashboard_collector = event_collector
+        # Store websocket manager for real-time event emission (Phase 1.2)
+        self._websocket_manager = websocket_manager
 
         # Inject event collector into all existing agents
         for agent in self._agents.values():
@@ -671,7 +684,11 @@ class Flock(metaclass=AutoTracedMeta):
         self.is_dashboard = is_dashboard
         # Only show banner in CLI mode, not dashboard mode
         if not self.is_dashboard:
-            init_console(clear_screen=True, show_banner=True, model=self.model)
+            try:
+                init_console(clear_screen=True, show_banner=True, model=self.model)
+            except (UnicodeEncodeError, UnicodeDecodeError):
+                # Skip banner on Windows consoles with encoding issues (e.g., tests, CI)
+                pass
         # Handle different input types
         if isinstance(obj, Artifact):
             # Already an artifact - publish as-is
@@ -881,10 +898,103 @@ class Flock(metaclass=AutoTracedMeta):
                     continue
                 if self._seen_before(artifact, agent):
                     continue
+
+                # JoinSpec CORRELATION: Check if subscription has correlated AND gate
+                if subscription.join is not None:
+                    # Use CorrelationEngine for JoinSpec (correlated AND gates)
+                    subscription_index = agent.subscriptions.index(subscription)
+                    completed_group = self._correlation_engine.add_artifact(
+                        artifact=artifact,
+                        subscription=subscription,
+                        subscription_index=subscription_index,
+                    )
+
+                    if completed_group is None:
+                        # Still waiting for correlation to complete
+                        # Phase 1.2: Emit real-time correlation update event
+                        await self._emit_correlation_updated_event(
+                            agent_name=agent.name,
+                            subscription_index=subscription_index,
+                            artifact=artifact,
+                        )
+                        continue
+
+                    # Correlation complete! Get all correlated artifacts
+                    artifacts = completed_group.get_artifacts()
+                else:
+                    # AND GATE LOGIC: Use artifact collector for simple AND gates (no correlation)
+                    is_complete, artifacts = self._artifact_collector.add_artifact(
+                        agent, subscription, artifact
+                    )
+
+                    if not is_complete:
+                        # Still waiting for more types (AND gate incomplete)
+                        continue
+
+                # BatchSpec BATCHING: Check if subscription has batch accumulator
+                if subscription.batch is not None:
+                    # Add to batch accumulator
+                    subscription_index = agent.subscriptions.index(subscription)
+
+                    # COMBINED FEATURES: JoinSpec + BatchSpec
+                    # If we have JoinSpec, artifacts is a correlated GROUP - treat as single batch item
+                    # If we have AND gate, artifacts is a complete set - treat as single batch item
+                    # Otherwise (single type), add each artifact individually
+
+                    if subscription.join is not None or len(subscription.type_models) > 1:
+                        # JoinSpec or AND gate: Treat artifact group as ONE batch item
+                        should_flush = self._batch_engine.add_artifact_group(
+                            artifacts=artifacts,
+                            subscription=subscription,
+                            subscription_index=subscription_index,
+                        )
+                    else:
+                        # Single type subscription: Add each artifact individually
+                        should_flush = False
+                        for single_artifact in artifacts:
+                            should_flush = self._batch_engine.add_artifact(
+                                artifact=single_artifact,
+                                subscription=subscription,
+                                subscription_index=subscription_index,
+                            )
+
+                            if should_flush:
+                                # Size threshold reached! Flush batch now
+                                break
+
+                    if not should_flush:
+                        # Batch not full yet - wait for more artifacts
+                        # Phase 1.2: Emit real-time batch update event
+                        await self._emit_batch_item_added_event(
+                            agent_name=agent.name,
+                            subscription_index=subscription_index,
+                            subscription=subscription,
+                            artifact=artifact,
+                        )
+                        continue
+
+                    # Flush the batch and get all accumulated artifacts
+                    batched_artifacts = self._batch_engine.flush_batch(
+                        agent.name, subscription_index
+                    )
+
+                    if batched_artifacts is None:
+                        # No batch to flush (shouldn't happen, but defensive)
+                        continue
+
+                    # Replace artifacts with batched artifacts
+                    artifacts = batched_artifacts
+
+                # Complete! Schedule agent with all collected artifacts
                 # T068: Increment iteration counter
                 self._agent_iteration_count[agent.name] = iteration_count + 1
-                self._mark_processed(artifact, agent)
-                self._schedule_task(agent, [artifact])
+
+                # Mark all artifacts as processed (prevent duplicate triggers)
+                for collected_artifact in artifacts:
+                    self._mark_processed(collected_artifact, agent)
+
+                # Schedule agent with ALL artifacts (batched, correlated, or AND gate complete)
+                self._schedule_task(agent, artifacts)
 
     def _schedule_task(self, agent: Agent, artifacts: list[Artifact]) -> None:
         task = asyncio.create_task(self._run_agent_task(agent, artifacts))
@@ -933,6 +1043,156 @@ class Flock(metaclass=AutoTracedMeta):
             except Exception as exc:  # pragma: no cover - defensive logging
                 self._logger.exception("Failed to record artifact consumption: %s", exc)
 
+    # Phase 1.2: Logic Operations Event Emission ----------------------------
+
+    async def _emit_correlation_updated_event(
+        self, *, agent_name: str, subscription_index: int, artifact: Artifact
+    ) -> None:
+        """Emit CorrelationGroupUpdatedEvent for real-time dashboard updates.
+
+        Called when an artifact is added to a correlation group that is not yet complete.
+
+        Args:
+            agent_name: Name of the agent with the JoinSpec subscription
+            subscription_index: Index of the subscription in the agent's subscriptions list
+            artifact: The artifact that triggered this update
+        """
+        # Only emit if dashboard is enabled
+        if self._websocket_manager is None:
+            return
+
+        # Import _get_correlation_groups helper from dashboard service
+        from flock.dashboard.service import _get_correlation_groups
+
+        # Get current correlation groups state from engine
+        groups = _get_correlation_groups(self._correlation_engine, agent_name, subscription_index)
+
+        if not groups:
+            return  # No groups to report (shouldn't happen, but defensive)
+
+        # Find the group that was just updated (match by last updated time or artifact ID)
+        # For now, we'll emit an event for the FIRST group that's still waiting
+        # In practice, the artifact we just added should be in one of these groups
+        for group_state in groups:
+            if not group_state["is_complete"]:
+                # Import CorrelationGroupUpdatedEvent
+                from flock.dashboard.events import CorrelationGroupUpdatedEvent
+
+                # Build and emit event
+                event = CorrelationGroupUpdatedEvent(
+                    agent_name=agent_name,
+                    subscription_index=subscription_index,
+                    correlation_key=group_state["correlation_key"],
+                    collected_types=group_state["collected_types"],
+                    required_types=group_state["required_types"],
+                    waiting_for=group_state["waiting_for"],
+                    elapsed_seconds=group_state["elapsed_seconds"],
+                    expires_in_seconds=group_state["expires_in_seconds"],
+                    expires_in_artifacts=group_state["expires_in_artifacts"],
+                    artifact_id=str(artifact.id),
+                    artifact_type=artifact.type,
+                    is_complete=group_state["is_complete"],
+                )
+
+                # Broadcast via WebSocket
+                await self._websocket_manager.broadcast(event)
+                break  # Only emit one event per artifact addition
+
+    async def _emit_batch_item_added_event(
+        self,
+        *,
+        agent_name: str,
+        subscription_index: int,
+        subscription: Subscription,  # noqa: F821
+        artifact: Artifact,
+    ) -> None:
+        """Emit BatchItemAddedEvent for real-time dashboard updates.
+
+        Called when an artifact is added to a batch that hasn't reached flush threshold.
+
+        Args:
+            agent_name: Name of the agent with the BatchSpec subscription
+            subscription_index: Index of the subscription in the agent's subscriptions list
+            subscription: The subscription with BatchSpec configuration
+            artifact: The artifact that triggered this update
+        """
+        # Only emit if dashboard is enabled
+        if self._websocket_manager is None:
+            return
+
+        # Import _get_batch_state helper from dashboard service
+        from flock.dashboard.service import _get_batch_state
+
+        # Get current batch state from engine
+        batch_state = _get_batch_state(
+            self._batch_engine, agent_name, subscription_index, subscription.batch
+        )
+
+        if not batch_state:
+            return  # No batch to report (shouldn't happen, but defensive)
+
+        # Import BatchItemAddedEvent
+        from flock.dashboard.events import BatchItemAddedEvent
+
+        # Build and emit event
+        event = BatchItemAddedEvent(
+            agent_name=agent_name,
+            subscription_index=subscription_index,
+            items_collected=batch_state["items_collected"],
+            items_target=batch_state.get("items_target"),
+            items_remaining=batch_state.get("items_remaining"),
+            elapsed_seconds=batch_state["elapsed_seconds"],
+            timeout_seconds=batch_state.get("timeout_seconds"),
+            timeout_remaining_seconds=batch_state.get("timeout_remaining_seconds"),
+            will_flush=batch_state["will_flush"],
+            artifact_id=str(artifact.id),
+            artifact_type=artifact.type,
+        )
+
+        # Broadcast via WebSocket
+        await self._websocket_manager.broadcast(event)
+
+    # Batch Helpers --------------------------------------------------------
+
+    async def _check_batch_timeouts(self) -> None:
+        """Check all batches for timeout expiry and flush expired batches.
+
+        This method is called periodically or manually (in tests) to enforce
+        timeout-based batching.
+        """
+        expired_batches = self._batch_engine.check_timeouts()
+
+        for agent_name, subscription_index in expired_batches:
+            # Flush the expired batch
+            artifacts = self._batch_engine.flush_batch(agent_name, subscription_index)
+
+            if artifacts is None:
+                continue
+
+            # Get the agent
+            agent = self._agents.get(agent_name)
+            if agent is None:
+                continue
+
+            # Schedule agent with batched artifacts
+            self._schedule_task(agent, artifacts)
+
+    async def _flush_all_batches(self) -> None:
+        """Flush all partial batches (for shutdown - ensures zero data loss)."""
+        all_batches = self._batch_engine.flush_all()
+
+        for agent_name, _subscription_index, artifacts in all_batches:
+            # Get the agent
+            agent = self._agents.get(agent_name)
+            if agent is None:
+                continue
+
+            # Schedule agent with partial batch
+            self._schedule_task(agent, artifacts)
+
+        # Wait for all scheduled tasks to complete
+        await self.run_until_idle()
+
     # Helpers --------------------------------------------------------------
 
     def _normalize_input(

flock/patches/__init__.py

@@ -2,4 +2,5 @@
 
 from flock.patches.dspy_streaming_patch import apply_patch, restore_original
 
+
 __all__ = ["apply_patch", "restore_original"]

flock/patches/dspy_streaming_patch.py

@@ -10,6 +10,7 @@ This patch replaces it with a non-blocking fire-and-forget approach.
 import asyncio
 import logging
 
+
 logger = logging.getLogger(__name__)
 
 

flock/subscription.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 from collections.abc import Callable, Iterable, Sequence
 from dataclasses import dataclass
+from datetime import timedelta
 from typing import TYPE_CHECKING, Any
 
 from pydantic import BaseModel
@@ -26,16 +27,68 @@ class TextPredicate:
 
 @dataclass
 class JoinSpec:
-    kind: str
-    window: float
-    by: Callable[[Artifact], Any] | None = None
+    """
+    Specification for correlated AND gates.
+
+    Correlates artifacts by a common key within a time OR count window.
+
+    Examples:
+        # Time-based correlation (within 5 minutes)
+        JoinSpec(
+            by=lambda x: x.correlation_id,
+            within=timedelta(minutes=5)
+        )
+
+        # Count-based correlation (within next 10 artifacts)
+        JoinSpec(
+            by=lambda x: x.correlation_id,
+            within=10
+        )
+
+    Args:
+        by: Callable that extracts the correlation key from an artifact payload
+        within: Window for correlation
+            - timedelta: Time window (artifacts must arrive within this time)
+            - int: Count window (artifacts must arrive within N published artifacts)
+    """
+
+    by: Callable[[BaseModel], Any]  # Extract correlation key from payload
+    within: timedelta | int  # Time window OR count window for correlation
 
 
 @dataclass
 class BatchSpec:
-    size: int
-    within: float
-    by: Callable[[Artifact], Any] | None = None
+    """
+    Specification for batch processing.
+
+    Accumulates artifacts and triggers agent when:
+    - Size threshold reached (e.g., batch of 10)
+    - Timeout expires (e.g., flush every 30 seconds)
+    - Whichever comes first
+
+    Examples:
+        # Size-based batching (flush when 25 artifacts accumulated)
+        BatchSpec(size=25)
+
+        # Timeout-based batching (flush every 30 seconds)
+        BatchSpec(timeout=timedelta(seconds=30))
+
+        # Hybrid (whichever comes first)
+        BatchSpec(size=100, timeout=timedelta(minutes=5))
+
+    Args:
+        size: Optional batch size threshold (flush when this many artifacts accumulated)
+        timeout: Optional timeout threshold (flush when this much time elapsed since first artifact)
+
+    Note: At least one of size or timeout must be specified.
+    """
+
+    size: int | None = None
+    timeout: timedelta | None = None
+
+    def __post_init__(self):
+        if self.size is None and self.timeout is None:
+            raise ValueError("BatchSpec requires at least one of: size, timeout")
 
 
 class Subscription:
@@ -60,7 +113,17 @@ class Subscription:
             raise ValueError("Subscription must declare at least one type.")
         self.agent_name = agent_name
         self.type_models: list[type[BaseModel]] = list(types)
-        self.type_names: set[str] = {type_registry.register(t) for t in types}
+
+        # Register all types and build counts (supports duplicates for count-based AND gates)
+        type_name_list = [type_registry.register(t) for t in types]
+        self.type_names: set[str] = set(type_name_list)  # Unique type names (for matching)
+
+        # Count-based AND gate: Track how many of each type are required
+        # Example: .consumes(A, A, B) → {"TypeA": 2, "TypeB": 1}
+        self.type_counts: dict[str, int] = {}
+        for type_name in type_name_list:
+            self.type_counts[type_name] = self.type_counts.get(type_name, 0) + 1
+
         self.where = list(where or [])
         self.text_predicates = list(text_predicates or [])
         self.from_agents = set(from_agents or [])

{flock_core-0.5.2.dist-info → flock_core-0.5.4.dist-info}/METADATA

@@ -1,6 +1,6 @@
 Metadata-Version: 2.4
 Name: flock-core
-Version: 0.5.2
+Version: 0.5.4
 Summary: Flock: A declrative framework for building and orchestrating AI agents.
 Author-email: Andre Ratzenberger <andre.ratzenberger@whiteduck.de>
 License: MIT
@@ -266,7 +266,7 @@ flock = Flock(os.getenv("DEFAULT_MODEL", "openai/gpt-4.1"))
 bug_detector = flock.agent("bug_detector").consumes(CodeSubmission).publishes(BugAnalysis)
 security_auditor = flock.agent("security_auditor").consumes(CodeSubmission).publishes(SecurityAnalysis)
 
-# This agent AUTOMATICALLY waits for both analyses
+# AND gate: This agent AUTOMATICALLY waits for BOTH analyses before triggering
 final_reviewer = flock.agent("final_reviewer").consumes(BugAnalysis, SecurityAnalysis).publishes(FinalReview)
 
 # 4. Run with real-time dashboard
@@ -343,29 +343,85 @@ analyzer = (
 )
 ```
 
-**Advanced subscriptions:**
+**Logic Operations (AND/OR Gates):**
+
+Flock provides intuitive syntax for coordinating multiple input types:
 
 ```python
-# Conditional consumption - only high-severity cases
+# AND gate: Wait for BOTH types before triggering
+diagnostician = flock.agent("diagnostician").consumes(XRayAnalysis, LabResults).publishes(Diagnosis)
+# Agent triggers only when both XRayAnalysis AND LabResults are available
+
+# OR gate: Trigger on EITHER type (via chaining)
+alert_handler = flock.agent("alerts").consumes(SystemAlert).consumes(UserAlert).publishes(Response)
+# Agent triggers when SystemAlert OR UserAlert is published
+
+# Count-based AND gate: Wait for MULTIPLE instances of the same type
+aggregator = flock.agent("aggregator").consumes(Order, Order, Order).publishes(BatchSummary)
+# Agent triggers when THREE Order artifacts are available
+
+# Mixed counts: Different requirements per type
+validator = flock.agent("validator").consumes(Image, Image, Metadata).publishes(ValidationResult)
+# Agent triggers when TWO Images AND ONE Metadata are available
+```
+
+**What just happened:**
+- ✅ **Natural syntax** - Code clearly expresses intent ("wait for 3 orders")
+- ✅ **Order-independent** - Artifacts can arrive in any sequence
+- ✅ **Latest wins** - If 4 As arrive but need 3, uses the 3 most recent
+- ✅ **Zero configuration** - No manual coordination logic needed
+
+**Advanced subscriptions unlock crazy powerful patterns:**
+
+<p align="center">
+  <img alt="Event Join" src="docs/assets/images/join.png" width="800">
+</p>
+
+```python
+# 🎯 Predicates - Smart filtering (only process critical cases)
 urgent_care = flock.agent("urgent").consumes(
     Diagnosis,
-    where=lambda d: d.severity in ["Critical", "High"]
+    where=lambda d: d.severity in ["Critical", "High"]  # Conditional routing!
 )
 
-# Batch processing - wait for 10 items
-batch_processor = flock.agent("batch").consumes(
-    Event,
-    batch=BatchSpec(size=10, timeout=timedelta(seconds=30))
+# 📦 BatchSpec - Cost optimization (process 10 at once = 90% cheaper API calls)
+payment_processor = flock.agent("payments").consumes(
+    Transaction,
+    batch=BatchSpec(size=25, timeout=timedelta(seconds=30))  # $5 saved per batch!
 )
 
-# Join operations - wait for multiple types within time window
-correlator = flock.agent("correlator").consumes(
-    SignalA,
-    SignalB,
-    join=JoinSpec(within=timedelta(minutes=5))
+# 🔗 JoinSpec - Data correlation (match orders + shipments by ID)
+customer_service = flock.agent("notifications").consumes(
+    Order,
+    Shipment,
+    join=JoinSpec(by=lambda x: x.order_id, within=timedelta(hours=24))  # Correlated!
+)
+
+# 🏭 Combined Features - Correlate sensors, THEN batch for analysis
+quality_control = flock.agent("qc").consumes(
+    TemperatureSensor,
+    PressureSensor,
+    join=JoinSpec(by=lambda x: x.device_id, within=timedelta(seconds=30)),
+    batch=BatchSpec(size=5, timeout=timedelta(seconds=45))  # IoT at scale!
 )
 ```
 
+**What just happened:**
+- ✅ **Predicates** route work by business rules ("only critical severity")
+- ✅ **BatchSpec** optimizes costs (25 transactions = 1 API call instead of 25)
+- ✅ **JoinSpec** correlates related data (orders ↔ shipments, sensors ↔ readings)
+- ✅ **Combined** delivers production-grade multi-stage pipelines
+
+**Real-world impact:**
+- 💰 E-commerce: Save $5 per batch on payment processing fees
+- 🏥 Healthcare: Correlate patient scans + lab results for diagnosis
+- 🏭 Manufacturing: Monitor 1000+ IoT sensors with efficient batching
+- 📊 Finance: Match trades + confirmations within 5-minute windows
+
+<p align="center">
+  <img alt="Event Batch" src="docs/assets/images/batch.png" width="800">
+</p>
+
 ### Visibility Controls (The Security)
 
 **Unlike other frameworks, Flock has zero-trust security built-in:**