devsper 2.1.6__py3-none-any.whl

devsper/bus/__init__.py

@@ -0,0 +1,29 @@
+"""Message bus: real pub/sub for task and agent events."""
+
+from devsper.bus.message import BusMessage, create_bus_message
+from devsper.bus.backends.base import BusBackend
+from devsper.bus.backends.memory import InMemoryBus
+from devsper.bus.backends.redis import RedisBus
+
+
+def get_bus(config: object) -> BusBackend:
+    """Return bus backend from config. Default InMemoryBus if bus config missing."""
+    backend = getattr(getattr(config, "bus", None), "backend", "memory")
+    if backend == "redis":
+        redis_url = getattr(getattr(config, "bus", None), "redis_url", "redis://localhost:6379")
+        return RedisBus(redis_url=redis_url)
+    return InMemoryBus()
+
+
+__all__ = [
+    "get_bus",
+    "MessageBus",
+    "BusMessage",
+    "BusBackend",
+    "InMemoryBus",
+    "RedisBus",
+    "create_bus_message",
+]
+
+# Alias for spec
+MessageBus = BusBackend

devsper/bus/backends/__init__.py

@@ -0,0 +1,5 @@
+from devsper.bus.backends.base import BusBackend
+from devsper.bus.backends.memory import InMemoryBus
+from devsper.bus.backends.redis import RedisBus
+
+__all__ = ["BusBackend", "InMemoryBus", "RedisBus"]

devsper/bus/backends/base.py

@@ -0,0 +1,38 @@
+"""Abstract base for message bus backends."""
+
+from abc import ABC, abstractmethod
+from typing import Awaitable, Callable
+
+from devsper.bus.message import BusMessage
+
+
+class BusBackend(ABC):
+    @abstractmethod
+    async def publish(self, message: BusMessage) -> None:
+        """Publish a message to the bus."""
+        ...
+
+    @abstractmethod
+    async def subscribe(
+        self,
+        topic: str,
+        handler: Callable[[BusMessage], Awaitable[None]],
+        run_id: str | None = None,
+    ) -> None:
+        """Subscribe to a topic (supports wildcards like task.*). run_id scopes channel when set (Redis)."""
+        ...
+
+    @abstractmethod
+    async def unsubscribe(self, topic: str) -> None:
+        """Unsubscribe from a topic."""
+        ...
+
+    @abstractmethod
+    async def start(self) -> None:
+        """Start the backend (e.g. connect to Redis)."""
+        ...
+
+    @abstractmethod
+    async def stop(self) -> None:
+        """Stop the backend."""
+        ...

devsper/bus/backends/memory.py

@@ -0,0 +1,55 @@
+"""In-memory asyncio bus backend. Single-node, zero config."""
+
+import asyncio
+import fnmatch
+from typing import Awaitable, Callable
+
+from devsper.bus.message import BusMessage
+from devsper.bus.backends.base import BusBackend
+
+
+def _topic_matches(pattern: str, topic: str) -> bool:
+    """Return True if topic matches pattern (supports * wildcard)."""
+    if pattern == topic:
+        return True
+    if "*" in pattern:
+        return fnmatch.fnmatch(topic, pattern)
+    return False
+
+
+class InMemoryBus(BusBackend):
+    """Asyncio-based in-memory bus. Dict of topic pattern -> list of handler coroutines."""
+
+    def __init__(self) -> None:
+        self._handlers: dict[str, list[Callable[[BusMessage], Awaitable[None]]]] = {}
+        self._started = False
+
+    async def start(self) -> None:
+        self._started = True
+
+    async def stop(self) -> None:
+        self._started = False
+        self._handlers.clear()
+
+    async def publish(self, message: BusMessage) -> None:
+        topic = message.topic
+        to_call: list[Awaitable[None]] = []
+        for pattern, handlers in self._handlers.items():
+            if _topic_matches(pattern, topic):
+                for h in handlers:
+                    to_call.append(h(message))
+        if to_call:
+            await asyncio.gather(*to_call)
+
+    async def subscribe(
+        self,
+        topic: str,
+        handler: Callable[[BusMessage], Awaitable[None]],
+        run_id: str | None = None,
+    ) -> None:
+        if topic not in self._handlers:
+            self._handlers[topic] = []
+        self._handlers[topic].append(handler)
+
+    async def unsubscribe(self, topic: str) -> None:
+        self._handlers.pop(topic, None)

devsper/bus/backends/redis.py

@@ -0,0 +1,146 @@
+"""Redis pub/sub bus backend."""
+
+import asyncio
+import logging
+from typing import Awaitable, Callable
+
+from devsper.bus.message import BusMessage
+from devsper.bus.backends.base import BusBackend
+from devsper.types.exceptions import BusConnectionError
+
+log = logging.getLogger(__name__)
+
+
+class RedisBus(BusBackend):
+    """Redis pub/sub backend. Lazy import redis.asyncio."""
+
+    def __init__(self, redis_url: str = "redis://localhost:6379") -> None:
+        self._redis_url = redis_url
+        self._pub: object = None
+        self._sub: object = None
+        self._pubsub: object = None
+        self._handlers: dict[str, list[Callable[[BusMessage], Awaitable[None]]]] = {}
+        self._listen_task: asyncio.Task | None = None
+        self._running = False
+
+    async def start(self) -> None:
+        try:
+            import redis.asyncio as aioredis
+        except ImportError as e:
+            raise ImportError(
+                "Redis bus requires redis package. Install with: pip install redis"
+            ) from e
+        try:
+            self._pub = aioredis.from_url(self._redis_url)
+            self._sub = aioredis.from_url(self._redis_url)
+            self._pubsub = self._sub.pubsub()
+            await self._pub.ping()
+        except Exception as e:
+            raise BusConnectionError(
+                f"Cannot connect to Redis at {self._redis_url}: {e}"
+            ) from e
+        self._running = True
+
+    @property
+    def redis_client(self) -> object:
+        """Redis connection for cluster registry/election/state (v1.10)."""
+        return self._pub
+
+    async def stop(self) -> None:
+        self._running = False
+        if self._listen_task is not None:
+            self._listen_task.cancel()
+            try:
+                await self._listen_task
+            except asyncio.CancelledError:
+                pass
+            self._listen_task = None
+        if self._pubsub is not None:
+            try:
+                await self._pubsub.aclose()
+            except Exception:
+                pass
+            self._pubsub = None
+        if self._pub is not None:
+            try:
+                await self._pub.aclose()
+            except Exception:
+                pass
+            self._pub = None
+        if self._sub is not None:
+            try:
+                await self._sub.aclose()
+            except Exception:
+                pass
+            self._sub = None
+        self._handlers.clear()
+
+    def _channel(self, topic: str, run_id: str | None = None) -> str:
+        """Redis channel: topic:run_id when run_id is set, else topic (all runs)."""
+        if run_id and run_id.strip():
+            return f"{topic}:{run_id}"
+        return topic
+
+    async def publish(self, message: BusMessage) -> None:
+        if self._pub is None:
+            raise BusConnectionError("Redis bus not started. Call start() first.")
+        channel = self._channel(message.topic, getattr(message, "run_id", None))
+        await self._pub.publish(channel, message.to_json())
+
+    async def subscribe(
+        self,
+        topic: str,
+        handler: Callable[[BusMessage], Awaitable[None]],
+        run_id: str | None = None,
+    ) -> None:
+        if self._pubsub is None:
+            raise BusConnectionError("Redis bus not started. Call start() first.")
+        channel = self._channel(topic, run_id)
+        if channel not in self._handlers:
+            self._handlers[channel] = []
+            await self._pubsub.subscribe(channel)
+        self._handlers[channel].append(handler)
+        if self._listen_task is None or self._listen_task.done():
+            self._listen_task = asyncio.create_task(self._listen())
+
+    async def _listen(self) -> None:
+        while self._running and self._pubsub is not None:
+            try:
+                async for raw in self._pubsub.listen():
+                    if not self._running:
+                        break
+                    if raw.get("type") == "message":
+                        channel = raw.get("channel")
+                        if isinstance(channel, bytes):
+                            channel = channel.decode("utf-8")
+                        data = raw.get("data")
+                        if data is None:
+                            continue
+                        payload = data.decode("utf-8") if isinstance(data, bytes) else data
+                        try:
+                            msg = BusMessage.from_json(payload)
+                            for h in self._handlers.get(channel, []):
+                                task = asyncio.create_task(h(msg))
+
+                                def _done(t):
+                                    try:
+                                        t.result()
+                                    except asyncio.CancelledError:
+                                        pass
+                                    except Exception as e:
+                                        log.warning("Bus handler failed: %s", e, exc_info=True)
+
+                                task.add_done_callback(_done)
+                        except Exception:
+                            pass
+            except asyncio.CancelledError:
+                break
+            except Exception:
+                if self._running:
+                    await asyncio.sleep(0.5)
+
+    async def unsubscribe(self, topic: str, run_id: str | None = None) -> None:
+        channel = self._channel(topic, run_id)
+        if self._pubsub is not None and channel in self._handlers:
+            await self._pubsub.unsubscribe(channel)
+        self._handlers.pop(channel, None)

devsper/bus/message.py

@@ -0,0 +1,56 @@
+"""Bus message: serializable pub/sub payload."""
+
+import json
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from uuid import uuid4
+
+
+@dataclass
+class BusMessage:
+    id: str
+    topic: str
+    payload: dict
+    sender_id: str
+    timestamp: str
+    run_id: str
+
+    def to_json(self) -> str:
+        return json.dumps({
+            "id": self.id,
+            "topic": self.topic,
+            "payload": self.payload,
+            "sender_id": self.sender_id,
+            "timestamp": self.timestamp,
+            "run_id": self.run_id,
+        })
+
+    @classmethod
+    def from_json(cls, raw: str) -> "BusMessage":
+        data = json.loads(raw)
+        return cls(
+            id=data["id"],
+            topic=data["topic"],
+            payload=dict(data.get("payload", {})),
+            sender_id=data.get("sender_id", "local"),
+            timestamp=data.get("timestamp", datetime.now(timezone.utc).isoformat()),
+            run_id=data.get("run_id", ""),
+        )
+
+
+def create_bus_message(
+    topic: str,
+    payload: dict,
+    *,
+    sender_id: str = "local",
+    run_id: str = "",
+) -> BusMessage:
+    """Create a BusMessage with generated id and current timestamp."""
+    return BusMessage(
+        id=str(uuid4()),
+        topic=topic,
+        payload=payload,
+        sender_id=sender_id,
+        timestamp=datetime.now(timezone.utc).isoformat(),
+        run_id=run_id,
+    )

devsper/bus/schema_version.py

@@ -0,0 +1,3 @@
+"""Bus message schema version; Rust worker publishes this in every BusMessage."""
+
+BUS_SCHEMA_VERSION = "1.10"

devsper/bus/topics.py

@@ -0,0 +1,19 @@
+"""Bus topic constants for pub/sub."""
+
+TASK_READY = "task.ready"
+TASK_STARTED = "task.started"
+TASK_COMPLETED = "task.completed"
+TASK_FAILED = "task.failed"
+TASK_CLAIMED = "task.claimed"
+TASK_CLAIM_GRANTED = "task.claim_granted"
+TASK_CLAIM_REJECTED = "task.claim_rejected"
+AGENT_BROADCAST = "agent.broadcast"
+SWARM_CONTROL = "swarm.control"
+NODE_HEARTBEAT = "node.heartbeat"
+NODE_JOINED = "node.joined"
+NODE_LEFT = "node.left"
+NODE_BECAME_LEADER = "node.became_leader"
+NODE_LOST_LEADERSHIP = "node.lost_leadership"
+SWARM_SNAPSHOT = "swarm.snapshot"
+SWARM_STATUS_REQUEST = "swarm.status_request"
+SWARM_STATUS_RESPONSE = "swarm.status_response"

devsper/cache/__init__.py

@@ -0,0 +1,6 @@
+"""Task result cache: hash task identity and store/retrieve results."""
+
+from devsper.cache.hashing import task_hash
+from devsper.cache.task_cache import TaskCache, SemanticTaskCache, CacheHit
+
+__all__ = ["task_hash", "TaskCache", "SemanticTaskCache", "CacheHit"]

devsper/cache/embedding_index.py

@@ -0,0 +1,98 @@
+"""
+In-memory embedding index for semantic cache lookup. Uses same embedding provider as memory.
+"""
+
+import io
+from typing import Any
+
+import numpy as np
+
+from devsper.memory.embeddings import embed_text
+
+
+def _cosine_sim(a: list[float], b: list[float]) -> float:
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    na = sum(x * x for x in a) ** 0.5
+    nb = sum(x * x for x in b) ** 0.5
+    if na == 0 or nb == 0:
+        return 0.0
+    return dot / (na * nb)
+
+
+def _embedding_to_bytes(vec: list[float]) -> bytes:
+    arr = np.array(vec, dtype=np.float64)
+    buf = io.BytesIO()
+    np.save(buf, arr, allow_pickle=False)
+    return buf.getvalue()
+
+
+def _bytes_to_embedding(data: bytes) -> list[float]:
+    buf = io.BytesIO(data)
+    arr = np.load(buf, allow_pickle=False)
+    return arr.tolist()
+
+
+class CacheEmbeddingIndex:
+    """
+    Same embedding provider as memory (embed_text). Loads entries from store,
+    keeps vectors in memory for nearest-neighbor search.
+    """
+
+    def __init__(self, store: Any = None) -> None:
+        self.store = store
+        self._vectors: list[list[float]] = []
+        self._meta: list[tuple[str, str, str, float, str]] = []  # key, result, task_type, created_at, original_description
+
+    def rebuild(self) -> None:
+        """Load all semantic entries from store and index by embedding."""
+        self._vectors = []
+        self._meta = []
+        if not self.store:
+            return
+        for emb_blob, result, task_type, created_at, key, original_desc in self.store.list_semantic_entries():
+            vec = _bytes_to_embedding(emb_blob)
+            self._vectors.append(vec)
+            self._meta.append((key, result, task_type, created_at, original_desc))
+
+    def add(self, embedding: list[float], key: str, result: str, task_type: str, created_at: float, original_description: str) -> None:
+        self._vectors.append(embedding)
+        self._meta.append((key, result, task_type, created_at, original_description))
+
+    def search(
+        self,
+        query_embedding: list[float],
+        threshold: float,
+        max_age_seconds: float,
+    ) -> tuple[float, str, str, str, str, str] | None:
+        """
+        Nearest neighbor search. Returns (similarity, result, original_description, cached_at, task_type, key)
+        for best match >= threshold and not expired, or None.
+        """
+        import time
+        now = time.time()
+        best_sim = -1.0
+        best = None
+        for i, vec in enumerate(self._vectors):
+            sim = _cosine_sim(query_embedding, vec)
+            if sim < threshold:
+                continue
+            key, result, task_type, created_at, original_desc = self._meta[i]
+            if max_age_seconds > 0 and (now - created_at) > max_age_seconds:
+                continue
+            if sim > best_sim:
+                best_sim = sim
+                best = (result, original_desc, created_at, task_type, key)
+        if best is None:
+            return None
+        result, original_desc, created_at, task_type, key = best
+        return (best_sim, result, original_desc, str(created_at), task_type, key)
+
+
+def embedding_to_bytes(vec: list[float]) -> bytes:
+    return _embedding_to_bytes(vec)
+
+
+def bytes_to_embedding(data: bytes) -> list[float]:
+    return _bytes_to_embedding(data)

devsper/cache/hashing.py

@@ -0,0 +1,24 @@
+"""Hash task identity for cache keys: description, dependencies, and optional tool usage."""
+
+import hashlib
+import json
+
+
+def task_hash(
+    task_id: str,
+    description: str,
+    dependencies: list[str],
+    tool_usage: str | None = None,
+) -> str:
+    """
+    Produce a stable hash for a task. Used for cache lookup.
+    Includes: task id, description, dependencies, and optional tool_usage signature.
+    """
+    payload = {
+        "id": task_id,
+        "description": (description or "").strip(),
+        "dependencies": sorted(dependencies) if dependencies else [],
+        "tool_usage": tool_usage or "",
+    }
+    blob = json.dumps(payload, sort_keys=True)
+    return hashlib.sha256(blob.encode("utf-8")).hexdigest()

devsper/cache/store.py

@@ -0,0 +1,153 @@
+"""
+Cache store backend for task cache. Holds exact-match and semantic (embedding) entries.
+SQLite with optional BLOB column for embeddings; semantic entries use a separate table.
+"""
+
+import sqlite3
+import time
+from pathlib import Path
+from typing import Protocol
+
+
+class CacheStore(Protocol):
+    """Protocol for cache storage (exact key or semantic)."""
+
+    def get(self, key: str) -> tuple[str, float] | None:
+        """Return (result, created_at) for key or None."""
+        ...
+
+    def set(self, key: str, result: str, created_at: float | None = None) -> None:
+        """Store result for key."""
+        ...
+
+    def list_semantic_entries(self) -> list[tuple[bytes, str, str, float, str, str]]:
+        """Return list of (embedding_blob, result, task_type, created_at, key, original_description)."""
+        ...
+
+    def put_semantic(
+        self,
+        key: str,
+        embedding_blob: bytes,
+        result: str,
+        task_type: str,
+        original_description: str,
+        created_at: float | None = None,
+    ) -> None:
+        """Store a semantic cache entry."""
+        ...
+
+    def delete(self, key: str) -> None:
+        """Remove entry by key."""
+        ...
+
+    def clear(self) -> None:
+        """Remove all entries."""
+        ...
+
+    def stats(self) -> dict:
+        """Return counts and optional semantic stats."""
+        ...
+
+
+def _default_db_path() -> Path:
+    return Path(".devsper") / "task_cache.db"
+
+
+class DefaultCacheStore:
+    """
+    SQLite-backed store: exact-match in task_cache; semantic in semantic_cache table.
+    """
+
+    def __init__(self, db_path: str | Path | None = None) -> None:
+        self.db_path = Path(db_path) if db_path else _default_db_path()
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._init_schema()
+
+    def _conn(self) -> sqlite3.Connection:
+        return sqlite3.connect(str(self.db_path))
+
+    def _init_schema(self) -> None:
+        with self._conn() as c:
+            c.execute("""
+                CREATE TABLE IF NOT EXISTS task_cache (
+                    key TEXT PRIMARY KEY,
+                    result TEXT NOT NULL,
+                    created_at REAL
+                )
+            """)
+            c.execute("""
+                CREATE TABLE IF NOT EXISTS semantic_cache (
+                    key TEXT PRIMARY KEY,
+                    embedding BLOB NOT NULL,
+                    result TEXT NOT NULL,
+                    task_type TEXT NOT NULL,
+                    created_at REAL NOT NULL,
+                    original_description TEXT NOT NULL
+                )
+            """)
+            c.execute(
+                "CREATE INDEX IF NOT EXISTS idx_semantic_task_type ON semantic_cache(task_type)"
+            )
+
+    def get(self, key: str) -> tuple[str, float] | None:
+        with self._conn() as c:
+            row = c.execute(
+                "SELECT result, created_at FROM task_cache WHERE key = ?", (key,)
+            ).fetchone()
+        return (row[0], row[1]) if row else None
+
+    def set(self, key: str, result: str, created_at: float | None = None) -> None:
+        ts = created_at if created_at is not None else time.time()
+        with self._conn() as c:
+            c.execute(
+                "INSERT OR REPLACE INTO task_cache (key, result, created_at) VALUES (?, ?, ?)",
+                (key, result, ts),
+            )
+
+    def list_semantic_entries(
+        self,
+    ) -> list[tuple[bytes, str, str, float, str, str]]:
+        with self._conn() as c:
+            rows = c.execute(
+                "SELECT embedding, result, task_type, created_at, key, original_description FROM semantic_cache"
+            ).fetchall()
+        return list(rows)
+
+    def put_semantic(
+        self,
+        key: str,
+        embedding_blob: bytes,
+        result: str,
+        task_type: str,
+        original_description: str,
+        created_at: float | None = None,
+    ) -> None:
+        ts = created_at if created_at is not None else time.time()
+        with self._conn() as c:
+            c.execute(
+                """INSERT OR REPLACE INTO semantic_cache
+                   (key, embedding, result, task_type, created_at, original_description)
+                   VALUES (?, ?, ?, ?, ?, ?)""",
+                (key, embedding_blob, result, task_type, ts, original_description),
+            )
+
+    def delete(self, key: str) -> None:
+        with self._conn() as c:
+            c.execute("DELETE FROM task_cache WHERE key = ?", (key,))
+            c.execute("DELETE FROM semantic_cache WHERE key = ?", (key,))
+
+    def clear(self) -> None:
+        with self._conn() as c:
+            c.execute("DELETE FROM task_cache")
+            c.execute("DELETE FROM semantic_cache")
+
+    def stats(self) -> dict:
+        with self._conn() as c:
+            exact = c.execute("SELECT COUNT(*) FROM task_cache").fetchone()[0]
+            semantic = c.execute("SELECT COUNT(*) FROM semantic_cache").fetchone()[0]
+        return {"entries": exact, "semantic_entries": semantic }
+
+
+def get_default_cache_store(db_path: str | Path | None = None) -> DefaultCacheStore:
+    """Return the default cache store (same DB path as TaskCache)."""
+    return DefaultCacheStore(db_path=db_path)