flock-core 0.5.11__py3-none-any.whl → 0.5.21__py3-none-any.whl

flock/agent/output_processor.py

@@ -0,0 +1,304 @@
+"""Agent output processing - validation, filtering, and artifact creation.
+
+Phase 4: Extracted from agent.py to eliminate C-rated complexity in _make_outputs_for_group.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from flock.core.artifacts import Artifact
+from flock.logging.logging import get_logger
+from flock.registry import type_registry
+from flock.utils.runtime import Context, EvalResult
+from flock.utils.type_resolution import TypeResolutionHelper
+
+
+if TYPE_CHECKING:
+    from flock.agent import AgentOutput, OutputGroup
+
+
+logger = get_logger(__name__)
+
+
+class OutputProcessor:
+    """Handles agent output validation, filtering, and artifact creation.
+
+    This module encapsulates all output processing logic including:
+    - Engine contract validation (expected vs actual artifact counts)
+    - WHERE filtering (reduces artifacts based on predicates)
+    - VALIDATE checks (fail-fast on validation errors)
+    - Dynamic visibility resolution
+    - Artifact matching and payload extraction
+    """
+
+    def __init__(self, agent_name: str):
+        """Initialize OutputProcessor for a specific agent.
+
+        Args:
+            agent_name: Name of the agent (for error messages and logging)
+        """
+        self._agent_name = agent_name
+        self._logger = logging.getLogger(__name__)
+
+    async def make_outputs_for_group(
+        self,
+        ctx: Context,
+        result: EvalResult,
+        output_group: OutputGroup,
+    ) -> list[Artifact]:
+        """Phase 3/5: Validate, filter, and create artifacts for specific OutputGroup.
+
+        This function:
+        1. Validates that the engine fulfilled its contract (produced expected count)
+        2. Applies WHERE filtering (reduces artifacts, no error)
+        3. Applies VALIDATE checks (raises ValueError if validation fails)
+        4. Applies visibility (static or dynamic)
+        5. Creates final artifacts with agent metadata
+
+        Args:
+            ctx: Context for this group
+            result: EvalResult from engine for THIS group
+            output_group: OutputGroup defining expected outputs
+
+        Returns:
+            List of artifacts matching this group's outputs
+
+        Raises:
+            ValueError: If engine violated contract or validation failed
+        """
+        produced: list[Artifact] = []
+
+        for output_decl in output_group.outputs:
+            # 1. Find ALL matching artifacts for this type
+            expected_canonical = type_registry.resolve_name(output_decl.spec.type_name)
+
+            matching_artifacts: list[Artifact] = []
+            for artifact in result.artifacts:
+                artifact_canonical = TypeResolutionHelper.safe_resolve(
+                    type_registry, artifact.type
+                )
+                if artifact_canonical == expected_canonical:
+                    matching_artifacts.append(artifact)
+
+            # 2. STRICT VALIDATION: Engine must produce exactly what was promised
+            # (This happens BEFORE filtering so engine contract is validated first)
+            expected_count = output_decl.count
+            actual_count = len(matching_artifacts)
+
+            if actual_count != expected_count:
+                raise ValueError(
+                    f"Engine contract violation in agent '{self._agent_name}': "
+                    f"Expected {expected_count} artifact(s) of type '{output_decl.spec.type_name}', "
+                    f"but engine produced {actual_count}. "
+                    f"Check your engine implementation to ensure it generates the correct number of outputs."
+                )
+
+            # 3. Apply WHERE filtering (Phase 5)
+            # Filtering reduces the number of published artifacts (this is intentional)
+            # NOTE: Predicates expect Pydantic model instances, not dicts
+            model_cls = type_registry.resolve(output_decl.spec.type_name)
+
+            if output_decl.filter_predicate:
+                original_count = len(matching_artifacts)
+                filtered = []
+                for a in matching_artifacts:
+                    # Reconstruct Pydantic model from payload dict
+                    model_instance = model_cls(**a.payload)
+                    if output_decl.filter_predicate(model_instance):
+                        filtered.append(a)
+                matching_artifacts = filtered
+                logger.debug(
+                    f"Agent {self._agent_name}: WHERE filter reduced artifacts from "
+                    f"{original_count} to {len(matching_artifacts)} for type {output_decl.spec.type_name}"
+                )
+
+            # 4. Apply VALIDATE checks (Phase 5)
+            # Validation failures raise errors (fail-fast)
+            if output_decl.validate_predicate:
+                if callable(output_decl.validate_predicate):
+                    # Single predicate
+                    for artifact in matching_artifacts:
+                        # Reconstruct Pydantic model from payload dict
+                        model_instance = model_cls(**artifact.payload)
+                        if not output_decl.validate_predicate(model_instance):
+                            raise ValueError(
+                                f"Validation failed for {output_decl.spec.type_name} "
+                                f"in agent '{self._agent_name}'"
+                            )
+                elif isinstance(output_decl.validate_predicate, list):
+                    # List of (callable, error_msg) tuples
+                    for artifact in matching_artifacts:
+                        # Reconstruct Pydantic model from payload dict
+                        model_instance = model_cls(**artifact.payload)
+                        for check, error_msg in output_decl.validate_predicate:
+                            if not check(model_instance):
+                                raise ValueError(
+                                    f"{error_msg}: {output_decl.spec.type_name}"
+                                )
+
+            # 5. Apply visibility and create artifacts (Phase 5)
+            for artifact_from_engine in matching_artifacts:
+                metadata = {
+                    "correlation_id": ctx.correlation_id,
+                    "artifact_id": artifact_from_engine.id,  # Preserve engine's ID
+                }
+
+                # Determine visibility (static or dynamic)
+                visibility = output_decl.default_visibility
+                if callable(visibility):
+                    # Dynamic visibility based on artifact content
+                    # Reconstruct Pydantic model from payload dict
+                    model_instance = model_cls(**artifact_from_engine.payload)
+                    visibility = visibility(model_instance)
+
+                # Override metadata visibility
+                metadata["visibility"] = visibility
+
+                # Re-wrap the artifact with agent metadata
+                artifact = output_decl.apply(
+                    artifact_from_engine.payload,
+                    produced_by=self._agent_name,
+                    metadata=metadata,
+                )
+                produced.append(artifact)
+                # Phase 6 SECURITY FIX: REMOVED publishing - orchestrator now handles it
+                # This fixes Vulnerability #2 (WRITE Bypass) - agents can no longer publish directly
+                # await ctx.board.publish(artifact)
+
+        return produced
+
+    async def make_outputs(
+        self, ctx: Context, result: EvalResult, output_groups: list[OutputGroup]
+    ) -> list[Artifact]:
+        """Output creation method for all output groups.
+
+        This method processes all output groups from a single engine evaluation.
+
+        Args:
+            ctx: Execution context
+            result: EvalResult from engine
+            output_groups: All output groups for the agent
+
+        Returns:
+            List of produced artifacts
+        """
+        if not output_groups:
+            # Utility agents may not publish anything
+            return list(result.artifacts)
+
+        produced: list[Artifact] = []
+
+        # For Phase 2: Iterate ALL output_groups (even though we only have 1 engine call)
+        # Phase 3 will modify this to call engine once PER group
+        for output_group in output_groups:
+            for output_decl in output_group.outputs:
+                # Phase 6: Find the matching artifact from engine result to preserve its ID
+                matching_artifact = self.find_matching_artifact(output_decl, result)
+
+                payload = self.select_payload(output_decl, result)
+                if payload is None:
+                    continue
+                metadata = {
+                    "correlation_id": ctx.correlation_id,
+                }
+
+                # Phase 6: Preserve artifact ID from engine (for streaming message preview)
+                if matching_artifact:
+                    metadata["artifact_id"] = matching_artifact.id
+
+                artifact = output_decl.apply(
+                    payload, produced_by=self._agent_name, metadata=metadata
+                )
+                produced.append(artifact)
+                # Phase 6: REMOVED publishing - orchestrator now handles it
+                # await ctx.board.publish(artifact)
+
+        return produced
+
+    def prepare_group_context(
+        self, ctx: Context, group_idx: int, output_group: OutputGroup
+    ) -> Context:
+        """Phase 3: Prepare context specific to this OutputGroup.
+
+        Creates a modified context for this group's engine call, potentially
+        with group-specific instructions or metadata.
+
+        Args:
+            ctx: Base context
+            group_idx: Index of this group (0-based)
+            output_group: The OutputGroup being processed
+
+        Returns:
+            Context for this group (may be the same instance or modified)
+        """
+        # For now, return the same context
+        # Phase 4 will add group-specific system prompts here
+        # Future: ctx.clone() and add group_description to system prompt
+        return ctx
+
+    def find_matching_artifact(
+        self, output_decl: AgentOutput, result: EvalResult
+    ) -> Artifact | None:
+        """Phase 6: Find artifact from engine result that matches this output declaration.
+
+        Returns the artifact object (with its ID) so we can preserve it when creating
+        the final published artifact. This ensures streaming events use the same ID.
+
+        Args:
+            output_decl: Output declaration to match
+            result: Engine result containing artifacts
+
+        Returns:
+            Matching artifact or None if not found
+        """
+        if not result.artifacts:
+            return None
+
+        # Normalize the expected type name to canonical form
+        expected_canonical = type_registry.resolve_name(output_decl.spec.type_name)
+
+        for artifact in result.artifacts:
+            # Normalize artifact type name to canonical form for comparison
+            artifact_canonical = TypeResolutionHelper.safe_resolve(
+                type_registry, artifact.type
+            )
+            if artifact_canonical == expected_canonical:
+                return artifact
+
+        return None
+
+    def select_payload(
+        self, output_decl: AgentOutput, result: EvalResult
+    ) -> dict[str, Any] | None:
+        """Extract payload from engine result for output declaration.
+
+        Args:
+            output_decl: Output declaration defining expected type
+            result: Engine result containing artifacts
+
+        Returns:
+            Payload dict or None if not found
+        """
+        # Normalize the expected type name to canonical form
+        expected_canonical = type_registry.resolve_name(output_decl.spec.type_name)
+
+        # Try to find payload in artifacts first
+        if result.artifacts:
+            for artifact in result.artifacts:
+                # Normalize artifact type name to canonical form for comparison
+                artifact_canonical = TypeResolutionHelper.safe_resolve(
+                    type_registry, artifact.type
+                )
+                if artifact_canonical == expected_canonical:
+                    return artifact.payload
+
+        # Fallback to state entries keyed by type name
+        maybe_data = result.state.get(output_decl.spec.type_name)
+        if isinstance(maybe_data, dict):
+            return maybe_data
+        return None
+
+
+__all__ = ["OutputProcessor"]

flock/api/__init__.py

@@ -0,0 +1,20 @@
+"""HTTP API service layer for Flock.
+
+This module contains HTTP service implementations and API models for
+serving the Flock orchestrator over HTTP with REST endpoints.
+"""
+
+from flock.api.models import (
+    ArtifactPublishRequest,
+    ArtifactPublishResponse,
+    CorrelationStatusResponse,
+)
+from flock.api.service import BlackboardHTTPService
+
+
+__all__ = [
+    "ArtifactPublishRequest",
+    "ArtifactPublishResponse",
+    "BlackboardHTTPService",
+    "CorrelationStatusResponse",
+]

flock/{api_models.py → api/models.py}

@@ -6,9 +6,7 @@ This improves API documentation and enables SDK generation.
 All models maintain 100% backwards compatibility with existing wire format.
 """
 
-from datetime import datetime
 from typing import Any, Literal
-from uuid import UUID
 
 from pydantic import BaseModel, Field
 

flock/{service.py → api/service.py}

@@ -10,7 +10,7 @@ from uuid import UUID
 from fastapi import FastAPI, HTTPException, Query
 from fastapi.responses import PlainTextResponse
 
-from flock.api_models import (
+from flock.api.models import (
     Agent,
     AgentListResponse,
     AgentRunRequest,
@@ -24,12 +24,12 @@ from flock.api_models import (
     HealthResponse,
     ProducedArtifact,
 )
+from flock.core.store import ArtifactEnvelope, ConsumptionRecord, FilterConfig
 from flock.registry import type_registry
-from flock.store import ArtifactEnvelope, ConsumptionRecord, FilterConfig
 
 
 if TYPE_CHECKING:
-    from flock.orchestrator import Flock
+    from flock.core import Flock
 
 
 class BlackboardHTTPService:

flock/cli.py

@@ -12,8 +12,8 @@ from typer.models import OptionInfo
 
 # Lazy import: only import examples when CLI commands are invoked
 # This prevents polluting type_registry on every package import
-from flock.service import BlackboardHTTPService
-from flock.store import SQLiteBlackboardStore
+from flock.api.service import BlackboardHTTPService
+from flock.core.store import SQLiteBlackboardStore
 
 
 app = typer.Typer(help="Blackboard Agents CLI")

flock/components/__init__.py

@@ -0,0 +1,41 @@
+"""Component library for extending Flock agents and orchestrators."""
+
+# Agent components
+from flock.components.agent import (
+    AgentComponent,
+    AgentComponentConfig,
+    EngineComponent,
+    OutputUtilityComponent,
+    OutputUtilityConfig,
+    TracedModelMeta,
+)
+
+# Orchestrator components
+from flock.components.orchestrator import (
+    BuiltinCollectionComponent,
+    CircuitBreakerComponent,
+    CollectionResult,
+    DeduplicationComponent,
+    OrchestratorComponent,
+    OrchestratorComponentConfig,
+    ScheduleDecision,
+)
+
+
+__all__ = [
+    # Agent components
+    "AgentComponent",
+    "AgentComponentConfig",
+    "EngineComponent",
+    "OutputUtilityComponent",
+    "OutputUtilityConfig",
+    "TracedModelMeta",
+    # Orchestrator components
+    "BuiltinCollectionComponent",
+    "CircuitBreakerComponent",
+    "CollectionResult",
+    "DeduplicationComponent",
+    "OrchestratorComponent",
+    "OrchestratorComponentConfig",
+    "ScheduleDecision",
+]

flock/components/agent/__init__.py

@@ -0,0 +1,22 @@
+"""Agent component library - Base classes and built-in components."""
+
+from flock.components.agent.base import (
+    AgentComponent,
+    AgentComponentConfig,
+    EngineComponent,
+    TracedModelMeta,
+)
+from flock.components.agent.output_utility import (
+    OutputUtilityComponent,
+    OutputUtilityConfig,
+)
+
+
+__all__ = [
+    "AgentComponent",
+    "AgentComponentConfig",
+    "EngineComponent",
+    "OutputUtilityComponent",
+    "OutputUtilityConfig",
+    "TracedModelMeta",
+]

flock/{components.py → components/agent/base.py}

@@ -1,4 +1,4 @@
-"""Agent component abstractions."""
+"""Agent component base classes and configuration."""
 
 from __future__ import annotations
 
@@ -13,8 +13,8 @@ from flock.logging.auto_trace import AutoTracedMeta
 
 if TYPE_CHECKING:  # pragma: no cover - type checking only
     from flock.agent import Agent, OutputGroup
-    from flock.artifacts import Artifact
-    from flock.runtime import Context, EvalInputs, EvalResult
+    from flock.core.artifacts import Artifact
+    from flock.utils.runtime import Context, EvalInputs, EvalResult
 
 T = TypeVar("T", bound="AgentComponentConfig")
 
@@ -219,4 +219,5 @@ __all__ = [
     "AgentComponent",
     "AgentComponentConfig",
     "EngineComponent",
+    "TracedModelMeta",
 ]

flock/{utility/output_utility_component.py → components/agent/output_utility.py}

@@ -1,25 +1,24 @@
-# src/flock/components/utility/output_utility_component.py
-"""Output formatting and display functionality for agents using unified component architecture."""
+"""Output formatting and display functionality for agents."""
 
 import re
 from typing import TYPE_CHECKING, Any
 
 from pydantic import Field
 
-from flock.components import AgentComponent, AgentComponentConfig
+from flock.components.agent.base import AgentComponent, AgentComponentConfig
 from flock.logging.formatters.themed_formatter import (
     ThemedAgentResultFormatter,
 )
 from flock.logging.formatters.themes import OutputTheme
 from flock.logging.logging import get_logger
-from flock.runtime import Context, EvalInputs, EvalResult
+from flock.utils.runtime import Context, EvalInputs, EvalResult
 
 
 if TYPE_CHECKING:  # pragma: no cover - type checking only
-    from flock.agent import Agent
+    from flock.core import Agent
 
 
-logger = get_logger("components.utility.output")
+logger = get_logger("components.agent.output_utility")
 
 
 class OutputUtilityConfig(AgentComponentConfig):
@@ -188,7 +187,7 @@ class OutputUtilityComponent(AgentComponent):
             if ctx:
                 import asyncio
 
-                from flock.agent import Agent
+                from flock.core import Agent
 
                 # Wait until no streams are active
                 max_wait = 30  # seconds
@@ -244,3 +243,9 @@ class OutputUtilityComponent(AgentComponent):
     def add_custom_formatter(self, key: str, formatter_name: str) -> None:
         """Add a custom formatter for a specific output key."""
         self.config.custom_formatters[key] = formatter_name
+
+
+__all__ = [
+    "OutputUtilityComponent",
+    "OutputUtilityConfig",
+]

flock/components/orchestrator/__init__.py

@@ -0,0 +1,22 @@
+"""Orchestrator component library - Base classes and built-in components."""
+
+from flock.components.orchestrator.base import (
+    CollectionResult,
+    OrchestratorComponent,
+    OrchestratorComponentConfig,
+    ScheduleDecision,
+)
+from flock.components.orchestrator.circuit_breaker import CircuitBreakerComponent
+from flock.components.orchestrator.collection import BuiltinCollectionComponent
+from flock.components.orchestrator.deduplication import DeduplicationComponent
+
+
+__all__ = [
+    "BuiltinCollectionComponent",
+    "CircuitBreakerComponent",
+    "CollectionResult",
+    "DeduplicationComponent",
+    "OrchestratorComponent",
+    "OrchestratorComponentConfig",
+    "ScheduleDecision",
+]