flock-core 0.5.20__py3-none-any.whl → 0.5.22__py3-none-any.whl

flock/semantic/context_provider.py

@@ -0,0 +1,173 @@
+"""Semantic context providers for agent execution.
+
+This module provides context providers that use semantic similarity to find
+relevant historical artifacts for agent context.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Callable
+from typing import TYPE_CHECKING, Any
+
+from pydantic import BaseModel
+
+
+if TYPE_CHECKING:
+    from flock.core.artifacts import Artifact
+    from flock.core.store import ArtifactStore
+
+
+class SemanticContextProvider:
+    """Context provider that retrieves semantically relevant historical artifacts.
+
+    This provider uses semantic similarity to find artifacts that are relevant
+    to a given query text, enabling agents to make decisions based on similar
+    past events.
+
+    Args:
+        query_text: The semantic query to match against artifacts
+        threshold: Minimum similarity score (0.0 to 1.0) to include in results
+        limit: Maximum number of artifacts to return
+        extract_field: Optional field name to extract from artifact payload for matching.
+                      If None, uses all text from payload.
+        artifact_type: Optional type filter - only return artifacts of this type
+        where: Optional predicate filter for additional filtering
+
+    Example:
+        ```python
+        provider = SemanticContextProvider(
+            query_text="user authentication issues", threshold=0.5, limit=5
+        )
+
+        relevant_artifacts = await provider.get_context(store)
+        ```
+    """
+
+    def __init__(
+        self,
+        query_text: str,
+        threshold: float = 0.4,
+        limit: int = 10,
+        extract_field: str | None = None,
+        artifact_type: type[BaseModel] | None = None,
+        where: Callable[[Artifact], bool] | None = None,
+    ):
+        """Initialize semantic context provider.
+
+        Args:
+            query_text: The semantic query text
+            threshold: Minimum similarity score (default: 0.4)
+            limit: Maximum results to return (default: 10)
+            extract_field: Optional field to extract from payload
+            artifact_type: Optional type filter
+            where: Optional predicate for additional filtering
+        """
+        if not query_text or not query_text.strip():
+            raise ValueError("query_text cannot be empty")
+
+        if not 0.0 <= threshold <= 1.0:
+            raise ValueError("threshold must be between 0 and 1")
+
+        if limit < 1:
+            raise ValueError("limit must be at least 1")
+
+        self.query_text = query_text
+        self.threshold = threshold
+        self.limit = limit
+        self.extract_field = extract_field
+        self.artifact_type = artifact_type
+        self.where = where
+
+    async def get_context(self, store: ArtifactStore) -> list[Artifact]:
+        """Retrieve semantically relevant artifacts from store.
+
+        Args:
+            store: The artifact store to query
+
+        Returns:
+            List of relevant artifacts, sorted by similarity (highest first)
+        """
+        # Check if semantic features available
+        try:
+            from flock.semantic import SEMANTIC_AVAILABLE, EmbeddingService
+        except ImportError:
+            return []
+
+        if not SEMANTIC_AVAILABLE:
+            return []
+
+        try:
+            embedding_service = EmbeddingService.get_instance()
+        except Exception:
+            return []
+
+        # Get query embedding
+        try:
+            query_embedding = embedding_service.embed(self.query_text)
+        except Exception:
+            return []
+
+        # Get all artifacts from store
+        all_artifacts = await store.list()
+
+        # Filter by type if specified
+        if self.artifact_type:
+            from flock.registry import type_registry
+
+            type_name = type_registry.register(self.artifact_type)
+            all_artifacts = [a for a in all_artifacts if a.type == type_name]
+
+        # Filter by where clause if specified
+        if self.where:
+            all_artifacts = [a for a in all_artifacts if self.where(a)]
+
+        # Compute similarities and filter
+        results: list[tuple[Artifact, float]] = []
+
+        for artifact in all_artifacts:
+            try:
+                # Extract text from artifact
+                if self.extract_field:
+                    # Use specific field
+                    text = str(artifact.payload.get(self.extract_field, ""))
+                else:
+                    # Use all text from payload
+                    text = self._extract_text_from_payload(artifact.payload)
+
+                if not text or not text.strip():
+                    continue
+
+                # Compute similarity
+                similarity = embedding_service.similarity(self.query_text, text)
+
+                # Check threshold
+                if similarity >= self.threshold:
+                    results.append((artifact, similarity))
+
+            except Exception:
+                # Skip artifacts that fail processing
+                continue
+
+        # Sort by similarity (highest first) and take top N
+        results.sort(key=lambda x: x[1], reverse=True)
+        return [artifact for artifact, _ in results[: self.limit]]
+
+    def _extract_text_from_payload(self, payload: dict[str, Any]) -> str:
+        """Extract all text content from payload.
+
+        Args:
+            payload: The artifact payload dict
+
+        Returns:
+            str: Concatenated text from all string fields
+        """
+        text_parts = []
+        for value in payload.values():
+            if isinstance(value, str):
+                text_parts.append(value)
+            elif isinstance(value, (list, tuple)):
+                for item in value:
+                    if isinstance(item, str):
+                        text_parts.append(item)
+
+        return " ".join(text_parts)

flock/semantic/embedding_service.py

@@ -0,0 +1,235 @@
+"""Embedding service for semantic matching.
+
+This module provides a singleton service for generating and caching embeddings
+using sentence-transformers.
+"""
+
+import logging
+from collections import OrderedDict
+
+import numpy as np
+
+
+logger = logging.getLogger(__name__)
+
+
+class LRUCache:
+    """Simple LRU cache with size limit."""
+
+    def __init__(self, max_size: int = 10000):
+        """Initialize LRU cache.
+
+        Args:
+            max_size: Maximum number of entries
+        """
+        self.max_size = max_size
+        self._cache: OrderedDict[str, np.ndarray] = OrderedDict()
+
+    def get(self, key: str) -> np.ndarray | None:
+        """Get value and mark as recently used."""
+        if key not in self._cache:
+            return None
+        # Move to end (most recent)
+        self._cache.move_to_end(key)
+        return self._cache[key]
+
+    def put(self, key: str, value: np.ndarray) -> None:
+        """Put value and evict LRU if needed."""
+        if key in self._cache:
+            # Update and move to end
+            self._cache.move_to_end(key)
+        self._cache[key] = value
+
+        # Evict oldest if over limit
+        if len(self._cache) > self.max_size:
+            self._cache.popitem(last=False)  # Remove oldest (first item)
+
+    def __contains__(self, key: str) -> bool:
+        """Check if key exists in cache."""
+        return key in self._cache
+
+    def __len__(self) -> int:
+        """Get cache size."""
+        return len(self._cache)
+
+
+class EmbeddingService:
+    """Singleton service for text embeddings using sentence-transformers.
+
+    This class manages the lifecycle of the embedding model and provides
+    efficient caching of embeddings.
+    """
+
+    _instance = None
+
+    def __init__(self, cache_size: int = 10000):
+        """Private constructor - use get_instance() instead.
+
+        Args:
+            cache_size: Maximum number of embeddings to cache
+        """
+        self._model = None
+        self._cache = LRUCache(max_size=cache_size)
+        self._cache_size = cache_size
+        self._hits = 0
+        self._misses = 0
+
+    @staticmethod
+    def get_instance(cache_size: int = 10000):
+        """Get or create the singleton EmbeddingService instance.
+
+        Args:
+            cache_size: Maximum number of embeddings to cache (default: 10000)
+
+        Returns:
+            EmbeddingService: The singleton instance
+        """
+        if EmbeddingService._instance is None:
+            EmbeddingService._instance = EmbeddingService(cache_size=cache_size)
+        return EmbeddingService._instance
+
+    def _load_model(self):
+        """Lazy load the sentence-transformers model."""
+        if self._model is None:
+            from sentence_transformers import SentenceTransformer
+
+            logger.info("Loading sentence-transformers model: all-MiniLM-L6-v2")
+            self._model = SentenceTransformer("all-MiniLM-L6-v2")
+            logger.info("Model loaded successfully")
+
+    def embed(self, text: str) -> np.ndarray:
+        """Generate embedding for a single text.
+
+        Args:
+            text: The text to embed
+
+        Returns:
+            np.ndarray: 384-dimensional embedding vector
+
+        Raises:
+            ValueError: If text is empty
+        """
+        if not text or not text.strip():
+            raise ValueError("Cannot embed empty text")
+
+        # Check cache first
+        cached = self._cache.get(text)
+        if cached is not None:
+            self._hits += 1
+            return cached
+
+        # Cache miss - generate embedding
+        self._misses += 1
+        self._load_model()
+
+        # Generate embedding
+        embedding = self._model.encode(
+            text, convert_to_numpy=True, show_progress_bar=False
+        )
+
+        # Ensure it's a float32 numpy array and flatten to 1D
+        if not isinstance(embedding, np.ndarray):
+            embedding = np.array(embedding, dtype=np.float32)
+
+        # Flatten to 1D if needed (model might return (1, 384) for single text)
+        if embedding.ndim > 1:
+            embedding = embedding.flatten()
+
+        # Store in cache
+        self._cache.put(text, embedding)
+
+        return embedding
+
+    def embed_batch(self, texts: list[str]) -> list[np.ndarray]:
+        """Generate embeddings for multiple texts efficiently.
+
+        Args:
+            texts: List of texts to embed
+
+        Returns:
+            list[np.ndarray]: List of embedding vectors
+        """
+        if not texts:
+            return []
+
+        # Separate cached and uncached
+        results = [None] * len(texts)
+        to_encode = []
+        to_encode_indices = []
+
+        for i, text in enumerate(texts):
+            cached = self._cache.get(text)
+            if cached is not None:
+                results[i] = cached
+                self._hits += 1
+            else:
+                to_encode.append(text)
+                to_encode_indices.append(i)
+                self._misses += 1
+
+        # Batch encode uncached texts
+        if to_encode:
+            self._load_model()
+            embeddings = self._model.encode(
+                to_encode, convert_to_numpy=True, show_progress_bar=False
+            )
+
+            # Store in cache and results
+            for i, (text, embedding) in enumerate(
+                zip(to_encode, embeddings, strict=False)
+            ):
+                if not isinstance(embedding, np.ndarray):
+                    embedding = np.array(embedding, dtype=np.float32)
+                # Flatten to 1D if needed
+                if embedding.ndim > 1:
+                    embedding = embedding.flatten()
+                self._cache.put(text, embedding)
+                results[to_encode_indices[i]] = embedding
+
+        return results  # type: ignore
+
+    def similarity(self, text1: str, text2: str) -> float:
+        """Compute semantic similarity between two texts.
+
+        Uses cosine similarity between embeddings.
+
+        Args:
+            text1: First text
+            text2: Second text
+
+        Returns:
+            float: Similarity score between 0 and 1
+        """
+        emb1 = self.embed(text1)
+        emb2 = self.embed(text2)
+
+        # Compute cosine similarity
+        dot_product = np.dot(emb1, emb2)
+        norm1 = np.linalg.norm(emb1)
+        norm2 = np.linalg.norm(emb2)
+
+        if norm1 == 0 or norm2 == 0:
+            return 0.0
+
+        similarity = dot_product / (norm1 * norm2)
+
+        # Clamp to [0, 1] and handle floating point errors
+        return float(max(0.0, min(1.0, similarity)))
+
+    def get_cache_stats(self) -> dict:
+        """Get cache hit/miss statistics.
+
+        Returns:
+            dict: Statistics including hits, misses, and hit rate
+        """
+        total = self._hits + self._misses
+        hit_rate = self._hits / total if total > 0 else 0.0
+
+        return {
+            "hits": self._hits,
+            "misses": self._misses,
+            "total": total,
+            "hit_rate": hit_rate,
+            "cache_size": len(self._cache),
+            "cache_limit": self._cache_size,
+        }