flock-core 0.5.10__py3-none-any.whl → 0.5.20__py3-none-any.whl

flock/agent/output_processor.py

@@ -0,0 +1,304 @@
+"""Agent output processing - validation, filtering, and artifact creation.
+
+Phase 4: Extracted from agent.py to eliminate C-rated complexity in _make_outputs_for_group.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING, Any
+
+from flock.core.artifacts import Artifact
+from flock.logging.logging import get_logger
+from flock.registry import type_registry
+from flock.utils.runtime import Context, EvalResult
+from flock.utils.type_resolution import TypeResolutionHelper
+
+
+if TYPE_CHECKING:
+    from flock.agent import AgentOutput, OutputGroup
+
+
+logger = get_logger(__name__)
+
+
+class OutputProcessor:
+    """Handles agent output validation, filtering, and artifact creation.
+
+    This module encapsulates all output processing logic including:
+    - Engine contract validation (expected vs actual artifact counts)
+    - WHERE filtering (reduces artifacts based on predicates)
+    - VALIDATE checks (fail-fast on validation errors)
+    - Dynamic visibility resolution
+    - Artifact matching and payload extraction
+    """
+
+    def __init__(self, agent_name: str):
+        """Initialize OutputProcessor for a specific agent.
+
+        Args:
+            agent_name: Name of the agent (for error messages and logging)
+        """
+        self._agent_name = agent_name
+        self._logger = logging.getLogger(__name__)
+
+    async def make_outputs_for_group(
+        self,
+        ctx: Context,
+        result: EvalResult,
+        output_group: OutputGroup,
+    ) -> list[Artifact]:
+        """Phase 3/5: Validate, filter, and create artifacts for specific OutputGroup.
+
+        This function:
+        1. Validates that the engine fulfilled its contract (produced expected count)
+        2. Applies WHERE filtering (reduces artifacts, no error)
+        3. Applies VALIDATE checks (raises ValueError if validation fails)
+        4. Applies visibility (static or dynamic)
+        5. Creates final artifacts with agent metadata
+
+        Args:
+            ctx: Context for this group
+            result: EvalResult from engine for THIS group
+            output_group: OutputGroup defining expected outputs
+
+        Returns:
+            List of artifacts matching this group's outputs
+
+        Raises:
+            ValueError: If engine violated contract or validation failed
+        """
+        produced: list[Artifact] = []
+
+        for output_decl in output_group.outputs:
+            # 1. Find ALL matching artifacts for this type
+            expected_canonical = type_registry.resolve_name(output_decl.spec.type_name)
+
+            matching_artifacts: list[Artifact] = []
+            for artifact in result.artifacts:
+                artifact_canonical = TypeResolutionHelper.safe_resolve(
+                    type_registry, artifact.type
+                )
+                if artifact_canonical == expected_canonical:
+                    matching_artifacts.append(artifact)
+
+            # 2. STRICT VALIDATION: Engine must produce exactly what was promised
+            # (This happens BEFORE filtering so engine contract is validated first)
+            expected_count = output_decl.count
+            actual_count = len(matching_artifacts)
+
+            if actual_count != expected_count:
+                raise ValueError(
+                    f"Engine contract violation in agent '{self._agent_name}': "
+                    f"Expected {expected_count} artifact(s) of type '{output_decl.spec.type_name}', "
+                    f"but engine produced {actual_count}. "
+                    f"Check your engine implementation to ensure it generates the correct number of outputs."
+                )
+
+            # 3. Apply WHERE filtering (Phase 5)
+            # Filtering reduces the number of published artifacts (this is intentional)
+            # NOTE: Predicates expect Pydantic model instances, not dicts
+            model_cls = type_registry.resolve(output_decl.spec.type_name)
+
+            if output_decl.filter_predicate:
+                original_count = len(matching_artifacts)
+                filtered = []
+                for a in matching_artifacts:
+                    # Reconstruct Pydantic model from payload dict
+                    model_instance = model_cls(**a.payload)
+                    if output_decl.filter_predicate(model_instance):
+                        filtered.append(a)
+                matching_artifacts = filtered
+                logger.debug(
+                    f"Agent {self._agent_name}: WHERE filter reduced artifacts from "
+                    f"{original_count} to {len(matching_artifacts)} for type {output_decl.spec.type_name}"
+                )
+
+            # 4. Apply VALIDATE checks (Phase 5)
+            # Validation failures raise errors (fail-fast)
+            if output_decl.validate_predicate:
+                if callable(output_decl.validate_predicate):
+                    # Single predicate
+                    for artifact in matching_artifacts:
+                        # Reconstruct Pydantic model from payload dict
+                        model_instance = model_cls(**artifact.payload)
+                        if not output_decl.validate_predicate(model_instance):
+                            raise ValueError(
+                                f"Validation failed for {output_decl.spec.type_name} "
+                                f"in agent '{self._agent_name}'"
+                            )
+                elif isinstance(output_decl.validate_predicate, list):
+                    # List of (callable, error_msg) tuples
+                    for artifact in matching_artifacts:
+                        # Reconstruct Pydantic model from payload dict
+                        model_instance = model_cls(**artifact.payload)
+                        for check, error_msg in output_decl.validate_predicate:
+                            if not check(model_instance):
+                                raise ValueError(
+                                    f"{error_msg}: {output_decl.spec.type_name}"
+                                )
+
+            # 5. Apply visibility and create artifacts (Phase 5)
+            for artifact_from_engine in matching_artifacts:
+                metadata = {
+                    "correlation_id": ctx.correlation_id,
+                    "artifact_id": artifact_from_engine.id,  # Preserve engine's ID
+                }
+
+                # Determine visibility (static or dynamic)
+                visibility = output_decl.default_visibility
+                if callable(visibility):
+                    # Dynamic visibility based on artifact content
+                    # Reconstruct Pydantic model from payload dict
+                    model_instance = model_cls(**artifact_from_engine.payload)
+                    visibility = visibility(model_instance)
+
+                # Override metadata visibility
+                metadata["visibility"] = visibility
+
+                # Re-wrap the artifact with agent metadata
+                artifact = output_decl.apply(
+                    artifact_from_engine.payload,
+                    produced_by=self._agent_name,
+                    metadata=metadata,
+                )
+                produced.append(artifact)
+                # Phase 6 SECURITY FIX: REMOVED publishing - orchestrator now handles it
+                # This fixes Vulnerability #2 (WRITE Bypass) - agents can no longer publish directly
+                # await ctx.board.publish(artifact)
+
+        return produced
+
+    async def make_outputs(
+        self, ctx: Context, result: EvalResult, output_groups: list[OutputGroup]
+    ) -> list[Artifact]:
+        """Output creation method for all output groups.
+
+        This method processes all output groups from a single engine evaluation.
+
+        Args:
+            ctx: Execution context
+            result: EvalResult from engine
+            output_groups: All output groups for the agent
+
+        Returns:
+            List of produced artifacts
+        """
+        if not output_groups:
+            # Utility agents may not publish anything
+            return list(result.artifacts)
+
+        produced: list[Artifact] = []
+
+        # For Phase 2: Iterate ALL output_groups (even though we only have 1 engine call)
+        # Phase 3 will modify this to call engine once PER group
+        for output_group in output_groups:
+            for output_decl in output_group.outputs:
+                # Phase 6: Find the matching artifact from engine result to preserve its ID
+                matching_artifact = self.find_matching_artifact(output_decl, result)
+
+                payload = self.select_payload(output_decl, result)
+                if payload is None:
+                    continue
+                metadata = {
+                    "correlation_id": ctx.correlation_id,
+                }
+
+                # Phase 6: Preserve artifact ID from engine (for streaming message preview)
+                if matching_artifact:
+                    metadata["artifact_id"] = matching_artifact.id
+
+                artifact = output_decl.apply(
+                    payload, produced_by=self._agent_name, metadata=metadata
+                )
+                produced.append(artifact)
+                # Phase 6: REMOVED publishing - orchestrator now handles it
+                # await ctx.board.publish(artifact)
+
+        return produced
+
+    def prepare_group_context(
+        self, ctx: Context, group_idx: int, output_group: OutputGroup
+    ) -> Context:
+        """Phase 3: Prepare context specific to this OutputGroup.
+
+        Creates a modified context for this group's engine call, potentially
+        with group-specific instructions or metadata.
+
+        Args:
+            ctx: Base context
+            group_idx: Index of this group (0-based)
+            output_group: The OutputGroup being processed
+
+        Returns:
+            Context for this group (may be the same instance or modified)
+        """
+        # For now, return the same context
+        # Phase 4 will add group-specific system prompts here
+        # Future: ctx.clone() and add group_description to system prompt
+        return ctx
+
+    def find_matching_artifact(
+        self, output_decl: AgentOutput, result: EvalResult
+    ) -> Artifact | None:
+        """Phase 6: Find artifact from engine result that matches this output declaration.
+
+        Returns the artifact object (with its ID) so we can preserve it when creating
+        the final published artifact. This ensures streaming events use the same ID.
+
+        Args:
+            output_decl: Output declaration to match
+            result: Engine result containing artifacts
+
+        Returns:
+            Matching artifact or None if not found
+        """
+        if not result.artifacts:
+            return None
+
+        # Normalize the expected type name to canonical form
+        expected_canonical = type_registry.resolve_name(output_decl.spec.type_name)
+
+        for artifact in result.artifacts:
+            # Normalize artifact type name to canonical form for comparison
+            artifact_canonical = TypeResolutionHelper.safe_resolve(
+                type_registry, artifact.type
+            )
+            if artifact_canonical == expected_canonical:
+                return artifact
+
+        return None
+
+    def select_payload(
+        self, output_decl: AgentOutput, result: EvalResult
+    ) -> dict[str, Any] | None:
+        """Extract payload from engine result for output declaration.
+
+        Args:
+            output_decl: Output declaration defining expected type
+            result: Engine result containing artifacts
+
+        Returns:
+            Payload dict or None if not found
+        """
+        # Normalize the expected type name to canonical form
+        expected_canonical = type_registry.resolve_name(output_decl.spec.type_name)
+
+        # Try to find payload in artifacts first
+        if result.artifacts:
+            for artifact in result.artifacts:
+                # Normalize artifact type name to canonical form for comparison
+                artifact_canonical = TypeResolutionHelper.safe_resolve(
+                    type_registry, artifact.type
+                )
+                if artifact_canonical == expected_canonical:
+                    return artifact.payload
+
+        # Fallback to state entries keyed by type name
+        maybe_data = result.state.get(output_decl.spec.type_name)
+        if isinstance(maybe_data, dict):
+            return maybe_data
+        return None
+
+
+__all__ = ["OutputProcessor"]

flock/api/__init__.py

@@ -0,0 +1,20 @@
+"""HTTP API service layer for Flock.
+
+This module contains HTTP service implementations and API models for
+serving the Flock orchestrator over HTTP with REST endpoints.
+"""
+
+from flock.api.models import (
+    ArtifactPublishRequest,
+    ArtifactPublishResponse,
+    CorrelationStatusResponse,
+)
+from flock.api.service import BlackboardHTTPService
+
+
+__all__ = [
+    "ArtifactPublishRequest",
+    "ArtifactPublishResponse",
+    "BlackboardHTTPService",
+    "CorrelationStatusResponse",
+]

flock/api/models.py

@@ -0,0 +1,283 @@
+"""Pydantic response models for Flock REST API.
+
+Provides proper OpenAPI schemas for all public API endpoints.
+This improves API documentation and enables SDK generation.
+
+All models maintain 100% backwards compatibility with existing wire format.
+"""
+
+from typing import Any, Literal
+
+from pydantic import BaseModel, Field
+
+
+# ============================================================================
+# Agent Models
+# ============================================================================
+
+
+class AgentSubscription(BaseModel):
+    """Subscription configuration for an agent."""
+
+    types: list[str] = Field(description="Artifact types this subscription consumes")
+    mode: str = Field(description="Subscription mode (e.g., 'all', 'any')")
+    delivery: str = Field(description="Delivery mode (e.g., 'immediate', 'batch')")
+
+
+class Agent(BaseModel):
+    """Single agent representation."""
+
+    name: str = Field(description="Unique name of the agent")
+    description: str = Field(default="", description="Human-readable description")
+    subscriptions: list[AgentSubscription] = Field(
+        description="List of subscriptions this agent listens to"
+    )
+    outputs: list[str] = Field(description="Artifact types this agent can produce")
+
+
+class AgentListResponse(BaseModel):
+    """Response for GET /api/v1/agents."""
+
+    agents: list[Agent] = Field(description="List of all registered agents")
+
+
+# ============================================================================
+# Artifact Models
+# ============================================================================
+
+
+class VisibilityInfo(BaseModel):
+    """Artifact visibility configuration."""
+
+    kind: str = Field(description="Visibility kind (e.g., 'Public', 'Private')")
+    # Additional visibility fields added dynamically
+
+
+class ArtifactBase(BaseModel):
+    """Base artifact representation with common fields."""
+
+    id: str = Field(description="Unique artifact identifier (UUID)")
+    type: str = Field(description="Artifact type name")
+    payload: dict[str, Any] = Field(description="Artifact payload data")
+    produced_by: str = Field(description="Name of agent/source that produced this")
+    visibility: dict[str, Any] = Field(description="Visibility configuration")
+    visibility_kind: str = Field(description="Visibility kind (Public/Private/etc)")
+    created_at: str = Field(
+        description="Timestamp when artifact was created (ISO 8601)"
+    )
+    correlation_id: str | None = Field(
+        None, description="Optional correlation ID for workflow tracking"
+    )
+    partition_key: str | None = Field(None, description="Optional partition key")
+    tags: list[str] = Field(default_factory=list, description="List of tags")
+    version: int = Field(description="Artifact version number")
+
+
+class ConsumptionRecord(BaseModel):
+    """Record of an artifact being consumed by an agent."""
+
+    artifact_id: str = Field(description="ID of the artifact that was consumed")
+    consumer: str = Field(description="Name of the agent that consumed it")
+    run_id: str = Field(description="Run ID of the consumption")
+    correlation_id: str = Field(description="Correlation ID of the consumption")
+    consumed_at: str = Field(description="Timestamp of consumption (ISO 8601)")
+
+
+class ArtifactWithConsumptions(ArtifactBase):
+    """Artifact with consumption metadata included."""
+
+    consumptions: list[ConsumptionRecord] = Field(
+        default_factory=list, description="List of consumption records"
+    )
+    consumed_by: list[str] = Field(
+        default_factory=list,
+        description="List of unique agent names that consumed this artifact",
+    )
+
+
+class PaginationInfo(BaseModel):
+    """Pagination metadata."""
+
+    limit: int = Field(description="Number of items per page")
+    offset: int = Field(description="Offset into the result set")
+    total: int = Field(description="Total number of items matching the query")
+
+
+class ArtifactListResponse(BaseModel):
+    """Response for GET /api/v1/artifacts."""
+
+    items: list[ArtifactBase | ArtifactWithConsumptions] = Field(
+        description="List of artifacts (may include consumption data if embed_meta=true)"
+    )
+    pagination: PaginationInfo = Field(description="Pagination information")
+
+
+class ArtifactPublishRequest(BaseModel):
+    """Request body for POST /api/v1/artifacts."""
+
+    type: str = Field(description="Artifact type name")
+    payload: dict[str, Any] = Field(
+        default_factory=dict, description="Artifact payload data"
+    )
+
+
+class ArtifactPublishResponse(BaseModel):
+    """Response for POST /api/v1/artifacts."""
+
+    status: Literal["accepted"] = Field(description="Publication status")
+
+
+class ArtifactSummary(BaseModel):
+    """Summary statistics for artifacts."""
+
+    # Define based on actual summary structure from store
+    # This is a placeholder - update based on actual implementation
+
+
+class ArtifactSummaryResponse(BaseModel):
+    """Response for GET /api/v1/artifacts/summary."""
+
+    summary: dict[str, Any] = Field(description="Summary statistics")
+
+
+# ============================================================================
+# Agent Run Models
+# ============================================================================
+
+
+class AgentRunInput(BaseModel):
+    """Input artifact for agent run."""
+
+    type: str = Field(description="Artifact type name")
+    payload: dict[str, Any] = Field(
+        default_factory=dict, description="Artifact payload data"
+    )
+
+
+class AgentRunRequest(BaseModel):
+    """Request body for POST /api/v1/agents/{name}/run."""
+
+    inputs: list[AgentRunInput] = Field(
+        default_factory=list, description="List of input artifacts"
+    )
+
+
+class ProducedArtifact(BaseModel):
+    """Artifact produced by agent run."""
+
+    id: str = Field(description="Artifact ID (UUID)")
+    type: str = Field(description="Artifact type name")
+    payload: dict[str, Any] = Field(description="Artifact payload data")
+    produced_by: str = Field(description="Name of agent that produced this")
+
+
+class AgentRunResponse(BaseModel):
+    """Response for POST /api/v1/agents/{name}/run."""
+
+    artifacts: list[ProducedArtifact] = Field(
+        description="Artifacts produced by the agent run"
+    )
+
+
+# ============================================================================
+# Schema Discovery Models
+# ============================================================================
+
+
+class ArtifactTypeSchema(BaseModel):
+    """Schema information for an artifact type."""
+
+    model_config = {"populate_by_name": True}  # Allow using 'schema' as field name
+
+    name: str = Field(description="Type name")
+    schema_: dict[str, Any] = Field(
+        alias="schema", description="JSON Schema for this type"
+    )
+
+
+class ArtifactTypesResponse(BaseModel):
+    """Response for GET /api/artifact-types."""
+
+    artifact_types: list[ArtifactTypeSchema] = Field(
+        description="List of all registered artifact types with their schemas"
+    )
+
+
+# ============================================================================
+# Agent History Models
+# ============================================================================
+
+
+class AgentHistorySummary(BaseModel):
+    """Summary of agent execution history."""
+
+    agent_id: str = Field(description="Agent identifier")
+    summary: dict[str, Any] = Field(description="History summary statistics")
+
+
+# ============================================================================
+# Correlation Status Models
+# ============================================================================
+
+
+class CorrelationStatusResponse(BaseModel):
+    """Response for GET /api/v1/correlations/{correlation_id}/status."""
+
+    correlation_id: str = Field(description="The correlation ID")
+    state: Literal["active", "completed", "failed", "not_found"] = Field(
+        description="Workflow state: active (work pending), completed (success), failed (only errors), not_found (no artifacts)"
+    )
+    has_pending_work: bool = Field(
+        description="Whether the orchestrator has pending work for this correlation"
+    )
+    artifact_count: int = Field(
+        description="Total number of artifacts with this correlation_id"
+    )
+    error_count: int = Field(description="Number of WorkflowError artifacts")
+    started_at: str | None = Field(
+        None, description="Timestamp of first artifact (ISO 8601)"
+    )
+    last_activity_at: str | None = Field(
+        None, description="Timestamp of most recent artifact (ISO 8601)"
+    )
+
+
+# ============================================================================
+# Health & Metrics Models
+# ============================================================================
+
+
+class HealthResponse(BaseModel):
+    """Response for GET /health."""
+
+    status: Literal["ok"] = Field(description="Health status")
+
+
+__all__ = [
+    # Agent models
+    "Agent",
+    "AgentSubscription",
+    "AgentListResponse",
+    # Artifact models
+    "ArtifactBase",
+    "ArtifactWithConsumptions",
+    "ArtifactListResponse",
+    "ArtifactPublishRequest",
+    "ArtifactPublishResponse",
+    "ArtifactSummaryResponse",
+    "PaginationInfo",
+    "ConsumptionRecord",
+    # Agent run models
+    "AgentRunRequest",
+    "AgentRunResponse",
+    "ProducedArtifact",
+    # Schema discovery
+    "ArtifactTypesResponse",
+    "ArtifactTypeSchema",
+    # History
+    "AgentHistorySummary",
+    # Correlation status
+    "CorrelationStatusResponse",
+    # Health
+    "HealthResponse",
+]