devsper 2.1.6__py3-none-any.whl

devsper/memory/embeddings.py

@@ -0,0 +1,60 @@
+"""
+Lightweight embeddings for memory index.
+
+Uses optional OpenAI embeddings when OPENAI_API_KEY is set;
+otherwise a deterministic stub for tests and offline use.
+"""
+
+import hashlib
+import os
+import re
+
+DEFAULT_DIM = 64
+
+
+def _stub_embed(text: str, dim: int = DEFAULT_DIM) -> list[float]:
+    """Deterministic pseudo-embedding from text (no API). Normalized to unit-ish scale."""
+    words = re.findall(r"\w+", text.lower())
+    if not words:
+        return [0.0] * dim
+    vec = [0.0] * dim
+    for w in words:
+        h = hashlib.sha256(w.encode()).hexdigest()
+        for i in range(0, min(dim * 2, len(h)), 2):
+            vec[i % dim] += int(h[i : i + 2], 16) / 255.0
+    total = sum(x * x for x in vec) ** 0.5
+    if total > 0:
+        vec = [x / total for x in vec]
+    return vec
+
+
+def _openai_embed(text: str, dim: int = DEFAULT_DIM) -> list[float] | None:
+    """Use OpenAI embeddings if available. Returns None if not configured."""
+    api_key = os.environ.get("OPENAI_API_KEY")
+    if not api_key or not text.strip():
+        return None
+    try:
+        from openai import OpenAI
+
+        client = OpenAI(api_key=api_key)
+        r = client.embeddings.create(
+            input=text[:8192],
+            model="text-embedding-3-small",
+        )
+        emb = r.data[0].embedding
+        if dim and len(emb) > dim:
+            return emb[:dim]
+        return emb
+    except Exception:
+        return None
+
+
+def embed_text(text: str, dim: int = DEFAULT_DIM) -> list[float]:
+    """
+    Return an embedding vector for text.
+    Uses OpenAI when OPENAI_API_KEY is set; otherwise a deterministic stub.
+    """
+    if not text or not text.strip():
+        return _stub_embed(" ", dim)
+    out = _openai_embed(text, dim)
+    return out if out is not None else _stub_embed(text, dim)

devsper/memory/memory_index.py

@@ -0,0 +1,97 @@
+"""
+Semantic search across stored memory via embeddings and top_k retrieval.
+"""
+
+from devsper.memory.embeddings import embed_text
+from devsper.memory.memory_store import MemoryStore
+from devsper.memory.memory_types import MemoryRecord
+
+
+def _cosine_sim(a: list[float], b: list[float]) -> float:
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b))
+    na = sum(x * x for x in a) ** 0.5
+    nb = sum(x * x for x in b) ** 0.5
+    if na == 0 or nb == 0:
+        return 0.0
+    return dot / (na * nb)
+
+
+class MemoryIndex:
+    """
+    Vector search over memory. Uses store for persistence and optional
+    embeddings on records for query_memory(text, top_k).
+    """
+
+    def __init__(self, store: MemoryStore | None = None) -> None:
+        self.store = store or MemoryStore()
+
+    def query_memory(
+        self,
+        text: str,
+        top_k: int = 5,
+        min_similarity: float = 0.0,
+        include_archived: bool = False,
+    ) -> list[MemoryRecord]:
+        """
+        Semantic search: embed query, score against stored records with embeddings,
+        return top_k by similarity above min_similarity. Records without embeddings
+        are skipped for ranking; if none have embeddings, return latest by timestamp.
+        Use min_similarity > 0 (e.g. 0.45) to avoid injecting barely-related memory.
+        By default excludes archived records (consolidation).
+        """
+        records = self.store.list_memory(limit=500, include_archived=include_archived)
+        if not records:
+            return []
+        query_emb = embed_text(text)
+        with_emb = [r for r in records if r.embedding is not None]
+        if not with_emb:
+            return records[:top_k]
+        scored = [
+            (_cosine_sim(query_emb, r.embedding), r)
+            for r in with_emb
+        ]
+        scored.sort(key=lambda x: -x[0])
+        if min_similarity > 0:
+            scored = [(s, r) for s, r in scored if s >= min_similarity]
+        return [r for _, r in scored[:top_k]]
+
+    def query_across_runs(
+        self,
+        text: str,
+        top_k: int = 20,
+        min_similarity: float = 0.0,
+        run_id_filter: str | None = None,
+        include_archived: bool = False,
+    ) -> list[MemoryRecord]:
+        """
+        v1.8: Same as query_memory but over more records (all runs), optional run_id filter.
+        Used by CrossRunSynthesizer. Excludes archived by default.
+        """
+        records = self.store.list_memory(
+            limit=2000,
+            include_archived=include_archived,
+            run_id_filter=run_id_filter,
+        )
+        if not records:
+            return []
+        query_emb = embed_text(text)
+        with_emb = [r for r in records if r.embedding is not None]
+        if not with_emb:
+            return records[:top_k]
+        scored = [
+            (_cosine_sim(query_emb, r.embedding), r)
+            for r in with_emb
+        ]
+        scored.sort(key=lambda x: -x[0])
+        if min_similarity > 0:
+            scored = [(s, r) for s, r in scored if s >= min_similarity]
+        return [r for _, r in scored[:top_k]]
+
+    def ensure_embedding(self, record: MemoryRecord) -> MemoryRecord:
+        """Compute and attach embedding if missing; return record (unchanged if already set)."""
+        if record.embedding is not None:
+            return record
+        record.embedding = embed_text(record.content)
+        return record

devsper/memory/memory_router.py

@@ -0,0 +1,62 @@
+"""
+Memory router: determine which memories are relevant to a task and return context for the agent.
+"""
+
+from devsper.memory.memory_index import MemoryIndex
+from devsper.memory.memory_store import MemoryStore
+from devsper.memory.memory_types import MemoryRecord
+
+
+class MemoryRouter:
+    """
+    Routes task descriptions to relevant memories (e.g. research, papers, codebase)
+    and formats them as context for the agent. Only memories above min_similarity
+    are included to avoid injecting off-topic context.
+    """
+
+    def __init__(
+        self,
+        store: MemoryStore | None = None,
+        index: MemoryIndex | None = None,
+        top_k: int = 10,
+        min_similarity: float = 0.55,
+    ) -> None:
+        self.store = store or MemoryStore()
+        self.index = index or MemoryIndex(self.store)
+        self.top_k = top_k
+        self.min_similarity = min_similarity
+
+    def get_relevant_memory(self, task: str) -> list[MemoryRecord]:
+        """
+        Return memories relevant to the task (semantic search).
+        Only returns records with similarity >= min_similarity to avoid off-topic injection.
+        """
+        return self.index.query_memory(
+            task,
+            top_k=self.top_k,
+            min_similarity=self.min_similarity,
+        )
+
+    def get_memory_context(self, task: str) -> str:
+        """
+        Format relevant memories as a string block for injection into the agent prompt.
+        User injections (tag user_injection) are always included first. Then semantic results.
+        Empty if no memories meet the relevance threshold.
+        """
+        lines = []
+        inject_records = self.store.list_memory(tag_contains="user_injection", limit=10)
+        if inject_records:
+            lines.append("USER INJECTIONS (high priority):")
+            for r in inject_records:
+                lines.append(f"- {r.content[:1000]}{'...' if len(r.content) > 1000 else ''}")
+        records = self.get_relevant_memory(task)
+        if records:
+            if lines:
+                lines.append("")
+            lines.append("RELEVANT MEMORY (previous research notes, findings, artifacts):")
+            for r in records:
+                lines.append(
+                    f"- [{r.memory_type.value}] {r.source_task or 'general'}: "
+                    f"{r.content[:500]}{'...' if len(r.content) > 500 else ''}"
+                )
+        return "\n".join(lines) if lines else ""

devsper/memory/memory_store.py

@@ -0,0 +1,221 @@
+"""
+Persistent memory store: SQLite-backed store, retrieve, delete, list.
+"""
+
+import json
+import os
+import sqlite3
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from devsper.memory.memory_types import MemoryRecord, MemoryType
+
+
+def _default_db_path() -> str:
+    from devsper.config import get_config
+
+    base = os.environ.get("DEVSPER_DATA_DIR") or get_config().data_dir
+    os.makedirs(base, exist_ok=True)
+    return os.path.join(base, "memory.db")
+
+
+class MemoryStore:
+    """Local persistent store for memory records. Uses SQLite."""
+
+    def __init__(self, db_path: str | None = None) -> None:
+        self.db_path = db_path or _default_db_path()
+        Path(self.db_path).parent.mkdir(parents=True, exist_ok=True)
+        self._init_schema()
+
+    def _conn(self) -> sqlite3.Connection:
+        return sqlite3.connect(self.db_path)
+
+    def _init_schema(self) -> None:
+        with self._conn() as conn:
+            conn.execute(
+                """
+                CREATE TABLE IF NOT EXISTS memory (
+                    memory_id TEXT PRIMARY KEY,
+                    memory_type TEXT NOT NULL,
+                    content TEXT NOT NULL,
+                    tags TEXT,
+                    timestamp TEXT NOT NULL,
+                    source_task TEXT,
+                    embedding TEXT
+                )
+                """
+            )
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS ix_memory_type ON memory(memory_type)"
+            )
+            conn.execute(
+                "CREATE INDEX IF NOT EXISTS ix_memory_timestamp ON memory(timestamp)"
+            )
+            try:
+                conn.execute("ALTER TABLE memory ADD COLUMN run_id TEXT DEFAULT ''")
+            except sqlite3.OperationalError:
+                pass
+            try:
+                conn.execute("ALTER TABLE memory ADD COLUMN archived INTEGER DEFAULT 0")
+            except sqlite3.OperationalError:
+                pass
+
+    def store(self, record: MemoryRecord) -> str:
+        """Store a memory record. Returns record id. Redacts PII if compliance.pii_redaction enabled."""
+        content = record.content
+        try:
+            from devsper.config import get_config
+            cfg = get_config()
+            if getattr(getattr(cfg, "compliance", None), "pii_redaction", False):
+                from devsper.compliance.pii import PIIRedactor
+                redactor = PIIRedactor(
+                    pii_types=getattr(cfg.compliance, "pii_types", None),
+                    gdpr_mode=getattr(cfg.compliance, "gdpr_mode", False),
+                )
+                res = redactor.redact(content or "")
+                content = res.redacted_text
+        except Exception:
+            pass
+        if content != record.content:
+            record = record.model_copy(update={"content": content})
+        row = record.to_store_row()
+        emb = row.get("embedding")
+        embedding_json = json.dumps(emb) if emb is not None else None
+        archived = row.get("archived", 0)
+        run_id = row.get("run_id", "") or ""
+        with self._conn() as conn:
+            conn.execute(
+                """
+                INSERT OR REPLACE INTO memory
+                (memory_id, memory_type, content, tags, timestamp, source_task, embedding, run_id, archived)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    row["memory_id"],
+                    row["memory_type"],
+                    row["content"],
+                    row["tags"],
+                    row["timestamp"],
+                    row["source_task"],
+                    embedding_json,
+                    run_id,
+                    archived,
+                ),
+            )
+        return row["memory_id"]
+
+    def retrieve(self, memory_id: str) -> MemoryRecord | None:
+        """Retrieve a single record by id."""
+        with self._conn() as conn:
+            conn.row_factory = sqlite3.Row
+            cur = conn.execute(
+                "SELECT memory_id, memory_type, content, tags, timestamp, source_task, embedding, run_id, archived FROM memory WHERE memory_id = ?",
+                (memory_id,),
+            )
+            row = cur.fetchone()
+        if row is None:
+            return None
+        return _row_to_record(dict(row))
+
+    def delete(self, memory_id: str) -> bool:
+        """Delete a record. Returns True if something was deleted."""
+        with self._conn() as conn:
+            cur = conn.execute("DELETE FROM memory WHERE memory_id = ?", (memory_id,))
+            return cur.rowcount > 0
+
+    def list_memory(
+        self,
+        memory_type: MemoryType | None = None,
+        limit: int = 100,
+        offset: int = 0,
+        tag_contains: str | None = None,
+        include_archived: bool = False,
+        run_id_filter: str | None = None,
+    ) -> list[MemoryRecord]:
+        """List records, optionally filtered by type, tag, archived, run_id, with limit/offset."""
+        with self._conn() as conn:
+            conn.row_factory = sqlite3.Row
+            conditions = []
+            params = []
+            if memory_type is not None:
+                conditions.append("memory_type = ?")
+                params.append(memory_type.value)
+            if tag_contains:
+                conditions.append("tags LIKE ?")
+                params.append(f"%{tag_contains}%")
+            if not include_archived:
+                conditions.append("COALESCE(archived, 0) = 0")
+            if run_id_filter is not None:
+                conditions.append("run_id = ?")
+                params.append(run_id_filter)
+            where = (" WHERE " + " AND ".join(conditions)) if conditions else ""
+            params.extend([limit, offset])
+            cur = conn.execute(
+                f"""
+                SELECT memory_id, memory_type, content, tags, timestamp, source_task, embedding,
+                       COALESCE(run_id, '') as run_id, COALESCE(archived, 0) as archived
+                FROM memory{where} ORDER BY timestamp DESC LIMIT ? OFFSET ?
+                """,
+                params,
+            )
+            rows = cur.fetchall()
+        return [_row_to_record(dict(r)) for r in rows]
+
+    def list_all_ids(self, memory_type: MemoryType | None = None) -> list[str]:
+        """List all memory ids (for index sync)."""
+        with self._conn() as conn:
+            if memory_type is not None:
+                cur = conn.execute(
+                    "SELECT memory_id FROM memory WHERE memory_type = ?",
+                    (memory_type.value,),
+                )
+            else:
+                cur = conn.execute("SELECT memory_id FROM memory")
+            return [r[0] for r in cur.fetchall()]
+
+    def set_archived(self, memory_id: str, archived: bool = True) -> bool:
+        """v1.8: Mark a record as archived (e.g. after consolidation)."""
+        with self._conn() as conn:
+            cur = conn.execute(
+                "UPDATE memory SET archived = ? WHERE memory_id = ?",
+                (1 if archived else 0, memory_id),
+            )
+            return cur.rowcount > 0
+
+
+def _row_to_record(row: dict) -> MemoryRecord:
+    tags_str = row.get("tags") or ""
+    tags = [t.strip() for t in tags_str.split(",") if t.strip()]
+    emb_raw = row.get("embedding")
+    embedding = json.loads(emb_raw) if isinstance(emb_raw, str) and emb_raw else None
+    archived = row.get("archived")
+    if archived is None and "archived" not in row:
+        archived = 0
+    return MemoryRecord(
+        id=row["memory_id"],
+        memory_type=MemoryType(row["memory_type"]),
+        content=row["content"],
+        tags=tags,
+        timestamp=datetime.fromisoformat(row["timestamp"]),
+        source_task=row.get("source_task") or "",
+        embedding=embedding,
+        run_id=row.get("run_id") or "",
+        archived=bool(archived) if isinstance(archived, (int, bool)) else False,
+    )
+
+
+def generate_memory_id() -> str:
+    """Generate a unique memory id."""
+    return str(uuid.uuid4())
+
+
+_default_store: MemoryStore | None = None
+
+
+def get_default_store() -> MemoryStore:
+    """Return the default process-wide memory store (for tools)."""
+    global _default_store
+    if _default_store is None:
+        _default_store = MemoryStore()
+    return _default_store

devsper/memory/memory_types.py

@@ -0,0 +1,54 @@
+"""
+Structured memory types for the swarm.
+
+Each memory record has: id, timestamp, source_task, content, tags, embedding (optional).
+"""
+
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Any
+
+from pydantic import BaseModel, Field
+
+
+class MemoryType(str, Enum):
+    """Kind of memory for routing and indexing."""
+
+    EPISODIC = "episodic"
+    SEMANTIC = "semantic"
+    ARTIFACT = "artifact"
+    RESEARCH = "research"
+
+
+class MemoryRecord(BaseModel):
+    """Single memory entry with optional embedding."""
+
+    id: str
+    memory_type: MemoryType
+    timestamp: datetime = Field(default_factory=lambda: datetime.now(timezone.utc))
+    source_task: str = ""
+    content: str
+    tags: list[str] = Field(default_factory=list)
+    embedding: list[float] | None = None
+    run_id: str = ""  # v1.8: which swarm run produced this (for cross-run synthesis)
+    archived: bool = False  # v1.8: consolidated into a summary record
+
+    def to_store_row(self) -> dict[str, Any]:
+        """Serialize for storage (embedding as JSON list or null)."""
+        return {
+            "memory_id": self.id,
+            "memory_type": self.memory_type.value,
+            "content": self.content,
+            "tags": ",".join(self.tags) if self.tags else "",
+            "timestamp": self.timestamp.isoformat(),
+            "source_task": self.source_task,
+            "embedding": self.embedding,
+            "run_id": getattr(self, "run_id", "") or "",
+            "archived": 1 if getattr(self, "archived", False) else 0,
+        }
+
+
+EpisodicMemory = MemoryRecord
+SemanticMemory = MemoryRecord
+ArtifactMemory = MemoryRecord
+ResearchMemory = MemoryRecord

devsper/memory/namespaces.py

@@ -0,0 +1,45 @@
+"""
+Memory namespaces: filter and tag memories by namespace (e.g. research_memory, coding_memory).
+Implemented via tags with prefix "ns:" for backward compatibility.
+"""
+
+from devsper.memory.memory_types import MemoryRecord
+
+NAMESPACE_TAG_PREFIX = "ns:"
+
+# Standard namespaces
+RESEARCH_MEMORY = "research_memory"
+CODING_MEMORY = "coding_memory"
+DATASET_MEMORY = "dataset_memory"
+
+DEFAULT_NAMESPACES = [RESEARCH_MEMORY, CODING_MEMORY, DATASET_MEMORY]
+
+
+def namespace_tag(namespace: str) -> str:
+    """Return the tag string for a namespace."""
+    return f"{NAMESPACE_TAG_PREFIX}{namespace}"
+
+
+def add_namespace(record: MemoryRecord, namespace: str) -> MemoryRecord:
+    """Add namespace tag to a record (returns new record with tag)."""
+    tag = namespace_tag(namespace)
+    tags = list(record.tags) if record.tags else []
+    if tag not in tags:
+        tags.append(tag)
+    return record.model_copy(update={"tags": tags})
+
+
+def record_namespace(record: MemoryRecord) -> str | None:
+    """Extract namespace from record tags, or None if none."""
+    if not record.tags:
+        return None
+    for t in record.tags:
+        if t.startswith(NAMESPACE_TAG_PREFIX):
+            return t[len(NAMESPACE_TAG_PREFIX) :]
+    return None
+
+
+def filter_by_namespace(records: list[MemoryRecord], namespace: str) -> list[MemoryRecord]:
+    """Return records that have the given namespace tag."""
+    tag = namespace_tag(namespace)
+    return [r for r in records if r.tags and tag in r.tags]

devsper/memory/scoring.py

@@ -0,0 +1,77 @@
+"""
+Memory scoring: relevance, recency, importance for ranking results.
+"""
+
+from datetime import datetime, timezone
+from math import exp
+
+from devsper.memory.memory_types import MemoryRecord
+
+
+def recency_score(record: MemoryRecord, now: datetime | None = None) -> float:
+    """Score by recency: newer = higher. Exponential decay."""
+    now = now or datetime.now(timezone.utc)
+    ts = record.timestamp
+    delta = (now - ts).total_seconds()
+    # Half-life ~ 1 day: score = 0.5 at 86400 seconds
+    half_life = 86400.0
+    return exp(-delta * (0.693 / half_life))
+
+
+def importance_score(record: MemoryRecord) -> float:
+    """Heuristic importance: longer content and more tags = slightly higher."""
+    base = 1.0
+    content_len = len(record.content or "")
+    if content_len > 500:
+        base += 0.2
+    if content_len > 2000:
+        base += 0.2
+    tag_count = len(record.tags or [])
+    if tag_count > 3:
+        base += 0.1
+    return min(base, 2.0)
+
+
+def combine_scores(
+    similarity: float,
+    recency: float,
+    importance: float,
+    similarity_weight: float = 0.7,
+    recency_weight: float = 0.2,
+    importance_weight: float = 0.1,
+) -> float:
+    """Combine normalized scores. Default: similarity dominates."""
+    return (
+        similarity_weight * similarity
+        + recency_weight * recency
+        + importance_weight * importance
+    )
+
+
+def score_and_sort(
+    records: list[MemoryRecord],
+    similarity_scores: list[float] | None = None,
+    now: datetime | None = None,
+) -> list[tuple[MemoryRecord, float]]:
+    """
+    Return (record, combined_score) sorted by score descending.
+    If similarity_scores is None, use 1.0 for all (recency + importance only).
+    """
+    if similarity_scores is None:
+        similarity_scores = [1.0] * len(records)
+    if len(similarity_scores) != len(records):
+        similarity_scores = [1.0] * len(records)
+    now = now or datetime.now(timezone.utc)
+    scored = [
+        (
+            r,
+            combine_scores(
+                similarity_scores[i],
+                recency_score(r, now),
+                importance_score(r),
+            ),
+        )
+        for i, r in enumerate(records)
+    ]
+    scored.sort(key=lambda x: -x[1])
+    return scored

devsper/memory/summarizer.py

@@ -0,0 +1,52 @@
+"""
+Memory summarization: summarize a set of memory records into shorter text.
+Optional LLM or extractive (first N chars / key sentences).
+"""
+
+from devsper.memory.memory_types import MemoryRecord
+
+MAX_EXTRACTIVE_CHARS = 4000
+
+
+def summarize_extractive(records: list[MemoryRecord], max_chars: int = MAX_EXTRACTIVE_CHARS) -> str:
+    """Concatenate record contents up to max_chars (no LLM)."""
+    parts: list[str] = []
+    total = 0
+    for r in records:
+        if total >= max_chars:
+            break
+        chunk = (r.content or "")[: max_chars - total]
+        if chunk:
+            parts.append(chunk)
+            total += len(chunk)
+    return "\n\n".join(parts) if parts else ""
+
+
+def summarize_with_llm(records: list[MemoryRecord], model_name: str = "gpt-4o-mini") -> str:
+    """Use LLM to summarize records. Returns summary or fallback to extractive on failure."""
+    text = summarize_extractive(records, max_chars=8000)
+    if not text.strip():
+        return ""
+    try:
+        from devsper.utils.models import generate
+        prompt = f"""Summarize the following memory entries into a concise summary (2-4 paragraphs).
+
+{text}
+
+Summary:"""
+        return generate(model_name, prompt).strip()
+    except Exception:
+        return summarize_extractive(records, max_chars=MAX_EXTRACTIVE_CHARS)
+
+
+def summarize(
+    records: list[MemoryRecord],
+    use_llm: bool = False,
+    model_name: str = "gpt-4o-mini",
+) -> str:
+    """Summarize records. use_llm=False uses extractive only."""
+    if not records:
+        return ""
+    if use_llm:
+        return summarize_with_llm(records, model_name=model_name)
+    return summarize_extractive(records)

devsper/nodes/__init__.py

@@ -0,0 +1,5 @@
+"""Distributed nodes: controller, worker, single-node."""
+
+from devsper.nodes.single import SingleNode
+
+__all__ = ["SingleNode"]