soothe-plugins 0.2.6__py3-none-any.whl

soothe_plugins/skillify/events.py

@@ -0,0 +1,148 @@
+"""Skillify subagent events.
+
+This module defines events for the skillify subagent.
+Events are self-registered at module load time.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+from pydantic import ConfigDict
+
+from soothe_sdk.core.events import SubagentEvent
+
+
+class SkillifyDispatchedEvent(SubagentEvent):
+    """Skillify subagent dispatched event."""
+
+    type: Literal["soothe.subagent.skillify.dispatched"] = "soothe.subagent.skillify.dispatched"
+    task: str = ""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class SkillifyCompletedEvent(SubagentEvent):
+    """Skillify subagent completed event."""
+
+    type: Literal["soothe.subagent.skillify.completed"] = "soothe.subagent.skillify.completed"
+    duration_ms: int = 0
+    result_count: int = 0
+
+    model_config = ConfigDict(extra="allow")
+
+
+class SkillifyIndexingPendingEvent(SubagentEvent):
+    """Skillify indexing pending event."""
+
+    type: Literal["soothe.subagent.skillify.indexing_pending"] = "soothe.subagent.skillify.indexing_pending"
+    query: str = ""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class SkillifyRetrieveStartedEvent(SubagentEvent):
+    """Skillify retrieve started event."""
+
+    type: Literal["soothe.subagent.skillify.retrieve_started"] = "soothe.subagent.skillify.retrieve_started"
+    query: str = ""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class SkillifyRetrieveCompletedEvent(SubagentEvent):
+    """Skillify retrieve completed event."""
+
+    type: Literal["soothe.subagent.skillify.retrieve_completed"] = "soothe.subagent.skillify.retrieve_completed"
+    query: str = ""
+    result_count: int = 0
+    top_score: float = 0.0
+
+    model_config = ConfigDict(extra="allow")
+
+
+class SkillifyRetrieveNotReadyEvent(SubagentEvent):
+    """Skillify retrieve not ready event."""
+
+    type: Literal["soothe.subagent.skillify.retrieve_not_ready"] = "soothe.subagent.skillify.retrieve_not_ready"
+    message: str = ""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class SkillifyIndexStartedEvent(SubagentEvent):
+    """Skillify index started event."""
+
+    type: Literal["soothe.subagent.skillify.index_started"] = "soothe.subagent.skillify.index_started"
+    collection: str = ""
+
+    model_config = ConfigDict(extra="allow")
+
+
+class SkillifyIndexUpdatedEvent(SubagentEvent):
+    """Skillify index updated event."""
+
+    type: Literal["soothe.subagent.skillify.index_updated"] = "soothe.subagent.skillify.index_updated"
+    new: int = 0
+    changed: int = 0
+    deleted: int = 0
+    total: int = 0
+
+    model_config = ConfigDict(extra="allow")
+
+
+class SkillifyIndexUnchangedEvent(SubagentEvent):
+    """Skillify index unchanged event."""
+
+    type: Literal["soothe.subagent.skillify.index_unchanged"] = "soothe.subagent.skillify.index_unchanged"
+    total: int = 0
+
+    model_config = ConfigDict(extra="allow")
+
+
+class SkillifyIndexFailedEvent(SubagentEvent):
+    """Skillify index failed event."""
+
+    type: Literal["soothe.subagent.skillify.index_failed"] = "soothe.subagent.skillify.index_failed"
+
+    model_config = ConfigDict(extra="allow")
+
+
+# Events are self-contained for community plugins.
+# Daemon will handle event registration based on type strings.
+# No explicit registration needed here.
+
+# Event type constants for convenient imports
+SUBAGENT_SKILLIFY_DISPATCHED = "soothe.subagent.skillify.dispatched"
+SUBAGENT_SKILLIFY_COMPLETED = "soothe.subagent.skillify.completed"
+SUBAGENT_SKILLIFY_INDEXING_PENDING = "soothe.subagent.skillify.indexing_pending"
+SUBAGENT_SKILLIFY_RETRIEVE_STARTED = "soothe.subagent.skillify.retrieve_started"
+SUBAGENT_SKILLIFY_RETRIEVE_COMPLETED = "soothe.subagent.skillify.retrieve_completed"
+SUBAGENT_SKILLIFY_RETRIEVE_NOT_READY = "soothe.subagent.skillify.retrieve_not_ready"
+SUBAGENT_SKILLIFY_INDEX_STARTED = "soothe.subagent.skillify.index_started"
+SUBAGENT_SKILLIFY_INDEX_UPDATED = "soothe.subagent.skillify.index_updated"
+SUBAGENT_SKILLIFY_INDEX_UNCHANGED = "soothe.subagent.skillify.index_unchanged"
+SUBAGENT_SKILLIFY_INDEX_FAILED = "soothe.subagent.skillify.index_failed"
+
+__all__ = [
+    "SUBAGENT_SKILLIFY_COMPLETED",
+    "SUBAGENT_SKILLIFY_DISPATCHED",
+    "SUBAGENT_SKILLIFY_INDEXING_PENDING",
+    "SUBAGENT_SKILLIFY_INDEX_FAILED",
+    "SUBAGENT_SKILLIFY_INDEX_STARTED",
+    "SUBAGENT_SKILLIFY_INDEX_UNCHANGED",
+    "SUBAGENT_SKILLIFY_INDEX_UPDATED",
+    "SUBAGENT_SKILLIFY_RETRIEVE_COMPLETED",
+    "SUBAGENT_SKILLIFY_RETRIEVE_NOT_READY",
+    "SUBAGENT_SKILLIFY_RETRIEVE_STARTED",
+    "SkillifyCompletedEvent",
+    "SkillifyDispatchedEvent",
+    "SkillifyIndexFailedEvent",
+    "SkillifyIndexStartedEvent",
+    "SkillifyIndexUnchangedEvent",
+    "SkillifyIndexUpdatedEvent",
+    "SkillifyIndexingPendingEvent",
+    "SkillifyRetrieveCompletedEvent",
+    "SkillifyRetrieveNotReadyEvent",
+    "SkillifyRetrieveStartedEvent",
+]

soothe_plugins/skillify/indexer.py

@@ -0,0 +1,312 @@
+"""SkillIndexer -- background loop for embedding and upserting skills (RFC-0004)."""
+
+from __future__ import annotations
+
+import asyncio
+import contextlib
+import logging
+from typing import TYPE_CHECKING, Any
+
+from .events import (
+    SkillifyIndexFailedEvent,
+    SkillifyIndexStartedEvent,
+    SkillifyIndexUnchangedEvent,
+    SkillifyIndexUpdatedEvent,
+)
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from langchain_core.embeddings import Embeddings
+
+from soothe_sdk.protocols import VectorStoreProtocol
+
+from .models import SkillRecord
+from .warehouse import SkillWarehouse
+
+logger = logging.getLogger(__name__)
+
+
+class LazyEmbeddings:
+    """Wrapper that creates fresh embedding instances per event loop."""
+
+    def __init__(self, factory: Callable[[], Embeddings]) -> None:
+        self._factory = factory
+        self._instances: dict[int, Embeddings] = {}
+
+    def _get_instance(self) -> Embeddings:
+        loop_id = id(asyncio.get_running_loop())
+        if loop_id not in self._instances:
+            self._instances[loop_id] = self._factory()
+        return self._instances[loop_id]
+
+    async def aembed_documents(self, texts: list[str]) -> list[list[float]]:
+        return await self._get_instance().aembed_documents(texts)
+
+    async def aembed_query(self, text: str) -> list[float]:
+        return await self._get_instance().aembed_query(text)
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        return self._get_instance().embed_documents(texts)
+
+    def embed_query(self, text: str) -> list[float]:
+        return self._get_instance().embed_query(text)
+
+
+class SkillIndexer:
+    """Background indexing loop that keeps the vector store in sync with the warehouse."""
+
+    def __init__(
+        self,
+        warehouse: SkillWarehouse,
+        vector_store: VectorStoreProtocol,
+        embeddings: Embeddings | Callable[[], Embeddings],
+        interval_seconds: int = 300,
+        collection: str = "soothe_skillify",
+        embedding_dims: int = 1536,
+        event_callback: Callable[[dict[str, Any]], None] | None = None,
+    ) -> None:
+        self._warehouse = warehouse
+        self._vector_store = vector_store
+        if callable(embeddings):
+            self._embeddings: Embeddings | LazyEmbeddings = LazyEmbeddings(embeddings)
+        else:
+            self._embeddings = embeddings
+        self._interval = interval_seconds
+        self._collection = collection
+        self._embedding_dims = embedding_dims
+        self._hash_cache: dict[str, str] = {}
+        self._task: asyncio.Task[None] | None = None
+        self._start_task: asyncio.Task[None] | None = None
+        self._initialized = False
+        self._total_indexed = 0
+        self._ready_event: asyncio.Event | None = None
+        self._event_callback = event_callback
+
+    @property
+    def total_indexed(self) -> int:
+        return self._total_indexed
+
+    @property
+    def ready_event(self) -> asyncio.Event:
+        if self._ready_event is None:
+            self._ready_event = asyncio.Event()
+        return self._ready_event
+
+    @property
+    def is_ready(self) -> bool:
+        if self._ready_event is None:
+            return False
+        return self._ready_event.is_set()
+
+    async def start(self) -> None:
+        if self._task is not None:
+            return
+        await self._ensure_collection()
+        await self._bootstrap_hash_cache()
+        self._emit(SkillifyIndexStartedEvent(collection=self._collection).to_dict())
+        self._task = asyncio.create_task(self._index_loop())
+        logger.info("Skillify background indexer started (interval=%ds)", self._interval)
+
+    async def stop(self) -> None:
+        if self._task is None:
+            return
+        self._task.cancel()
+        with contextlib.suppress(asyncio.CancelledError):
+            await self._task
+        self._task = None
+
+        if hasattr(self._vector_store, "close"):
+            try:
+                await self._vector_store.close()
+            except Exception:
+                logger.debug("Failed to close vector store", exc_info=True)
+
+        logger.info("Skillify background indexer stopped")
+
+    async def run_once(self) -> dict[str, int]:
+        stats: dict[str, int] = {"new": 0, "changed": 0, "deleted": 0}
+
+        current_records = self._warehouse.scan()
+        current_ids = {r.id for r in current_records}
+
+        to_embed: list[SkillRecord] = []
+        for record in current_records:
+            cached_hash = self._hash_cache.get(record.id)
+            if cached_hash is None:
+                to_embed.append(record)
+                stats["new"] += 1
+            elif cached_hash != record.content_hash:
+                to_embed.append(record)
+                stats["changed"] += 1
+
+        deleted_ids = set(self._hash_cache.keys()) - current_ids
+        for did in deleted_ids:
+            try:
+                await self._vector_store.delete(did)
+            except Exception:
+                logger.warning("Failed to delete stale record %s", did, exc_info=True)
+            self._hash_cache.pop(did, None)
+            stats["deleted"] += 1
+
+        if to_embed:
+            await self._embed_and_upsert(to_embed)
+
+        for record in current_records:
+            self._hash_cache[record.id] = record.content_hash
+
+        self._total_indexed = len(current_ids)
+        return stats
+
+    async def _embed_and_upsert(self, records: list[SkillRecord]) -> None:
+        texts = [self._embedding_text(r) for r in records]
+
+        sanitized_texts = []
+        for i, text in enumerate(texts):
+            sanitized_text = text
+            if not isinstance(sanitized_text, str):
+                logger.warning("Text %d is not a string (type=%s), converting", i, type(sanitized_text).__name__)
+                sanitized_text = str(sanitized_text) if sanitized_text is not None else "Untitled skill"
+            if not sanitized_text or not sanitized_text.strip():
+                logger.warning("Text %d is empty, using placeholder", i)
+                sanitized_text = "Untitled skill"
+            sanitized_texts.append(sanitized_text)
+
+        if not sanitized_texts:
+            logger.error("No valid texts for embedding")
+            return
+
+        try:
+            vectors = await self._embeddings.aembed_documents(sanitized_texts)
+        except Exception:
+            logger.exception(
+                "Embedding generation failed for %d skills. First text sample: %.200s",
+                len(records),
+                sanitized_texts[0] if sanitized_texts else "N/A",
+            )
+            return
+
+        payloads: list[dict[str, Any]] = []
+        ids: list[str] = []
+        for record in records:
+            payloads.append(
+                {
+                    "skill_id": record.id,
+                    "name": record.name,
+                    "description": record.description,
+                    "path": record.path,
+                    "tags": record.tags,
+                    "content_hash": record.content_hash,
+                }
+            )
+            ids.append(record.id)
+
+        try:
+            await self._vector_store.insert(vectors=vectors, payloads=payloads, ids=ids)
+        except Exception:
+            logger.exception("Vector store upsert failed for %d skills", len(records))
+
+    @staticmethod
+    def _embedding_text(record: SkillRecord) -> str:
+        parts = []
+        if record.name:
+            name_str = str(record.name).strip()
+            if name_str:
+                parts.append(name_str)
+        if record.description:
+            desc_str = str(record.description).strip()
+            if desc_str:
+                parts.append(desc_str)
+        if record.tags and isinstance(record.tags, (list, tuple)):
+            tag_strs = []
+            for tag in record.tags:
+                if tag is not None:
+                    tag_str = str(tag).strip()
+                    if tag_str:
+                        tag_strs.append(tag_str)
+            if tag_strs:
+                parts.append("Tags: " + ", ".join(tag_strs))
+
+        result = "\n".join(parts)
+        return result if result.strip() else "Untitled skill"
+
+    async def _ensure_collection(self) -> None:
+        if self._initialized:
+            return
+        try:
+            await self._vector_store.create_collection(
+                vector_size=self._embedding_dims,
+                distance="cosine",
+            )
+            self._initialized = True
+        except Exception:
+            logger.warning("Collection creation failed (may already exist)", exc_info=True)
+            self._initialized = True
+
+    async def _index_loop(self) -> None:
+        first_pass = True
+        while True:
+            try:
+                stats = await self.run_once()
+                total_changes = stats["new"] + stats["changed"] + stats["deleted"]
+                if total_changes > 0:
+                    self._emit(
+                        SkillifyIndexUpdatedEvent(
+                            new=stats["new"],
+                            changed=stats["changed"],
+                            deleted=stats["deleted"],
+                            total=self._total_indexed,
+                        ).to_dict()
+                    )
+                    logger.info(
+                        "Skillify index pass: new=%d changed=%d deleted=%d total=%d",
+                        stats["new"],
+                        stats["changed"],
+                        stats["deleted"],
+                        self._total_indexed,
+                    )
+                else:
+                    self._emit(
+                        SkillifyIndexUnchangedEvent(
+                            total=self._total_indexed,
+                        ).to_dict()
+                    )
+                    logger.debug("Skillify index pass: no changes (total=%d)", self._total_indexed)
+                if first_pass:
+                    if self._ready_event:
+                        self._ready_event.set()
+                    first_pass = False
+                    logger.info("Skillify index ready (total=%d)", self._total_indexed)
+            except asyncio.CancelledError:
+                raise
+            except Exception:
+                self._emit(SkillifyIndexFailedEvent().to_dict())
+                logger.exception("Skillify index pass failed")
+                if first_pass:
+                    if self._ready_event:
+                        self._ready_event.set()
+                    first_pass = False
+
+            await asyncio.sleep(self._interval)
+
+    async def _bootstrap_hash_cache(self) -> None:
+        try:
+            records = await self._vector_store.list_records(limit=10000)
+        except Exception:
+            logger.debug("Skillify hash cache bootstrap failed", exc_info=True)
+            return
+
+        for record in records:
+            payload = record.payload or {}
+            skill_id = payload.get("skill_id")
+            content_hash = payload.get("content_hash")
+            if isinstance(skill_id, str) and isinstance(content_hash, str) and skill_id and content_hash:
+                self._hash_cache[skill_id] = content_hash
+
+    def _emit(self, event: dict[str, Any]) -> None:
+        if self._event_callback is None:
+            return
+        try:
+            self._event_callback(event)
+        except Exception:
+            logger.debug("Skillify event callback failed", exc_info=True)

soothe_plugins/skillify/models.py

@@ -0,0 +1,36 @@
+"""Skillify data models (RFC-0004)."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+from typing import Literal
+
+from pydantic import BaseModel, Field
+
+
+class SkillRecord(BaseModel):
+    """Metadata for a single indexed skill."""
+
+    id: str
+    name: str
+    description: str
+    path: str
+    tags: list[str] = Field(default_factory=list)
+    status: Literal["indexed", "stale", "error"] = "indexed"
+    indexed_at: datetime = Field(default_factory=lambda: datetime.now(UTC))
+    content_hash: str = ""
+
+
+class SkillSearchResult(BaseModel):
+    """A single result from a retrieval query."""
+
+    record: SkillRecord
+    score: float
+
+
+class SkillBundle(BaseModel):
+    """Response payload for a retrieval request."""
+
+    query: str
+    results: list[SkillSearchResult] = Field(default_factory=list)
+    total_indexed: int = 0

soothe_plugins/skillify/retriever.py

@@ -0,0 +1,165 @@
+"""SkillRetriever -- semantic search over the skill index (RFC-0004)."""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import UTC, datetime
+from typing import TYPE_CHECKING, Any
+
+from .models import SkillBundle, SkillRecord, SkillSearchResult
+
+if TYPE_CHECKING:
+    from collections.abc import Callable
+
+    from langchain_core.embeddings import Embeddings
+
+from soothe_sdk.protocols import (
+    ActionRequest,
+    PermissionSet,
+    PolicyContext,
+    VectorStoreProtocol,
+)
+
+logger = logging.getLogger(__name__)
+
+_INDEXING_WAIT_TIMEOUT = 10.0
+
+
+class LazyEmbeddings:
+    """Wrapper that creates fresh embedding instances per event loop."""
+
+    def __init__(self, factory: Callable[[], Embeddings]) -> None:
+        self._factory = factory
+        self._instances: dict[int, Embeddings] = {}
+
+    def _get_instance(self) -> Embeddings:
+        loop_id = id(asyncio.get_running_loop())
+        if loop_id not in self._instances:
+            self._instances[loop_id] = self._factory()
+        return self._instances[loop_id]
+
+    async def aembed_documents(self, texts: list[str]) -> list[list[float]]:
+        return await self._get_instance().aembed_documents(texts)
+
+    async def aembed_query(self, text: str) -> list[float]:
+        return await self._get_instance().aembed_query(text)
+
+    def embed_documents(self, texts: list[str]) -> list[list[float]]:
+        return self._get_instance().embed_documents(texts)
+
+    def embed_query(self, text: str) -> list[float]:
+        return self._get_instance().embed_query(text)
+
+
+class SkillRetriever:
+    """Semantic search over the Skillify vector index."""
+
+    def __init__(
+        self,
+        vector_store: VectorStoreProtocol,
+        embeddings: Embeddings | Callable[[], Embeddings],
+        top_k: int = 10,
+        ready_event: asyncio.Event | None = None,
+        policy: Any | None = None,
+        policy_profile: str = "standard",
+    ) -> None:
+        self._vector_store = vector_store
+        if callable(embeddings):
+            self._embeddings: Embeddings | LazyEmbeddings = LazyEmbeddings(embeddings)
+        else:
+            self._embeddings = embeddings
+        self._top_k = top_k
+        self._ready_event = ready_event
+        self._policy = policy
+        self._policy_profile = policy_profile
+
+    @property
+    def is_ready(self) -> bool:
+        if self._ready_event is None:
+            return True
+        return self._ready_event.is_set()
+
+    async def retrieve(self, query: str, top_k: int | None = None) -> SkillBundle:
+        self._check_policy(query)
+
+        if self._ready_event and not self._ready_event.is_set():
+            logger.info("Skillify index not ready, waiting up to %.0fs", _INDEXING_WAIT_TIMEOUT)
+            try:
+                await asyncio.wait_for(self._ready_event.wait(), timeout=_INDEXING_WAIT_TIMEOUT)
+            except TimeoutError:
+                logger.warning("Skillify index still not ready after %.0fs timeout", _INDEXING_WAIT_TIMEOUT)
+                return SkillBundle(
+                    query="[Indexing in progress] The skill warehouse is still being indexed. Please retry shortly.",
+                )
+
+        k = top_k or self._top_k
+
+        try:
+            vector = await self._embeddings.aembed_query(query)
+        except Exception:
+            logger.exception("Query embedding failed for: %s", query[:100])
+            return SkillBundle(query=query)
+
+        try:
+            records = await self._vector_store.search(
+                query=query,
+                vector=vector,
+                limit=k,
+            )
+        except Exception:
+            logger.exception("Vector store search failed")
+            return SkillBundle(query=query)
+
+        results: list[SkillSearchResult] = []
+        for vr in records:
+            payload = vr.payload
+            record = SkillRecord(
+                id=payload.get("skill_id", vr.id),
+                name=payload.get("name", "unknown"),
+                description=payload.get("description", ""),
+                path=payload.get("path", ""),
+                tags=payload.get("tags", []),
+                status="indexed",
+                indexed_at=datetime.now(UTC),
+                content_hash=payload.get("content_hash", ""),
+            )
+            results.append(SkillSearchResult(record=record, score=vr.score or 0.0))
+
+        total_records = await self._count_indexed()
+
+        return SkillBundle(
+            query=query,
+            results=results,
+            total_indexed=total_records,
+        )
+
+    def _check_policy(self, query: str) -> None:
+        if self._policy is None:
+            return
+
+        permissions = PermissionSet(frozenset())
+        get_profile = getattr(self._policy, "get_profile", None)
+        if callable(get_profile):
+            profile = get_profile(self._policy_profile)
+            if profile is not None:
+                permissions = profile.permissions
+
+        decision = self._policy.check(
+            ActionRequest(
+                action_type="skillify_retrieve",
+                tool_name="skillify.retrieve",
+                tool_args={"query": query[:200]},
+            ),
+            PolicyContext(active_permissions=permissions, thread_id=None),
+        )
+        if decision.verdict == "deny":
+            msg = f"Policy denied skill retrieval: {decision.reason}"
+            raise ValueError(msg)
+
+    async def _count_indexed(self) -> int:
+        try:
+            all_records = await self._vector_store.list_records(limit=10000)
+            return len(all_records)
+        except Exception:
+            return 0