pyagent-context 0.1.0__py3-none-any.whl

pyagent_context/memory/semantic.py

@@ -0,0 +1,153 @@
+"""SemanticMemory: vector-indexed long-term store (Protocol + in-memory TF-IDF impl)."""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+from dataclasses import dataclass, field
+from typing import Protocol, runtime_checkable
+
+from pyagent_context.item import ContextItem
+
+
+@dataclass(frozen=True)
+class SearchResult:
+    """Result of a semantic search.
+
+    Attributes:
+        item: The matching context item.
+        score: Relevance score (0.0–1.0).
+    """
+
+    item: ContextItem
+    score: float
+
+
+@runtime_checkable
+class SemanticMemoryProtocol(Protocol):
+    """Interface for any vector-backed semantic memory store."""
+
+    def add(self, item: ContextItem) -> None:
+        """Index an item for later retrieval."""
+        ...
+
+    def search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+        """Find the most relevant items for a query."""
+        ...
+
+    def remove(self, item_id: str) -> bool:
+        """Remove an item by ID. Returns True if found."""
+        ...
+
+    def clear(self) -> None:
+        """Remove all items."""
+        ...
+
+    def __len__(self) -> int: ...
+
+
+class InMemorySemanticStore:
+    """In-memory semantic store using TF-IDF cosine similarity.
+
+    No external dependencies — suitable for testing and small datasets.
+    For production, use a vector DB adapter (ChromaDB, Pinecone, etc.).
+
+    Args:
+        stop_words: Optional set of words to ignore in scoring.
+    """
+
+    def __init__(self, stop_words: set[str] | None = None) -> None:
+        self._items: dict[str, ContextItem] = {}
+        self._tf_cache: dict[str, dict[str, float]] = {}
+        self._stop_words = stop_words or {
+            "a", "an", "the", "is", "are", "was", "were", "be", "been",
+            "being", "have", "has", "had", "do", "does", "did", "will",
+            "would", "could", "should", "may", "might", "can", "shall",
+            "to", "of", "in", "for", "on", "with", "at", "by", "from",
+            "as", "into", "through", "during", "before", "after", "and",
+            "but", "or", "nor", "not", "so", "yet", "both", "either",
+            "neither", "each", "every", "all", "any", "few", "more",
+            "most", "other", "some", "such", "no", "only", "own", "same",
+            "than", "too", "very", "just", "because", "about", "between",
+            "it", "its", "this", "that", "these", "those", "i", "me",
+            "my", "we", "our", "you", "your", "he", "him", "his", "she",
+            "her", "they", "them", "their", "what", "which", "who",
+        }
+
+    def add(self, item: ContextItem) -> None:
+        self._items[item.id] = item
+        self._tf_cache[item.id] = self._compute_tf(item.content)
+
+    def search(self, query: str, top_k: int = 5) -> list[SearchResult]:
+        if not self._items:
+            return []
+
+        query_tf = self._compute_tf(query)
+        idf = self._compute_idf()
+
+        query_tfidf = {w: tf * idf.get(w, 0.0) for w, tf in query_tf.items()}
+
+        results: list[SearchResult] = []
+        for item_id, item in self._items.items():
+            doc_tf = self._tf_cache.get(item_id, {})
+            doc_tfidf = {w: tf * idf.get(w, 0.0) for w, tf in doc_tf.items()}
+            score = self._cosine_similarity(query_tfidf, doc_tfidf)
+            if score > 0:
+                results.append(SearchResult(item=item, score=score))
+
+        results.sort(key=lambda r: r.score, reverse=True)
+        return results[:top_k]
+
+    def remove(self, item_id: str) -> bool:
+        if item_id in self._items:
+            del self._items[item_id]
+            self._tf_cache.pop(item_id, None)
+            return True
+        return False
+
+    def clear(self) -> None:
+        self._items.clear()
+        self._tf_cache.clear()
+
+    def __len__(self) -> int:
+        return len(self._items)
+
+    # ------------------------------------------------------------------
+    # TF-IDF helpers
+    # ------------------------------------------------------------------
+
+    def _tokenize(self, text: str) -> list[str]:
+        words = text.lower().split()
+        return [w.strip(".,!?;:\"'()[]{}") for w in words if w.strip(".,!?;:\"'()[]{}") not in self._stop_words]
+
+    def _compute_tf(self, text: str) -> dict[str, float]:
+        tokens = self._tokenize(text)
+        if not tokens:
+            return {}
+        counts = Counter(tokens)
+        total = len(tokens)
+        return {word: count / total for word, count in counts.items()}
+
+    def _compute_idf(self) -> dict[str, float]:
+        n_docs = len(self._items)
+        if n_docs == 0:
+            return {}
+        doc_freq: Counter[str] = Counter()
+        for tf in self._tf_cache.values():
+            for word in tf:
+                doc_freq[word] += 1
+        return {word: math.log(n_docs / (1 + freq)) for word, freq in doc_freq.items()}
+
+    @staticmethod
+    def _cosine_similarity(a: dict[str, float], b: dict[str, float]) -> float:
+        if not a or not b:
+            return 0.0
+        common_keys = set(a) & set(b)
+        if not common_keys:
+            return 0.0
+        dot = sum(a[k] * b[k] for k in common_keys)
+        mag_a = math.sqrt(sum(v ** 2 for v in a.values()))
+        mag_b = math.sqrt(sum(v ** 2 for v in b.values()))
+        if mag_a == 0 or mag_b == 0:
+            return 0.0
+        return dot / (mag_a * mag_b)

pyagent_context/memory/session.py

@@ -0,0 +1,151 @@
+"""SessionMemory: persisted across turns via JSON file or SQLite."""
+
+from __future__ import annotations
+
+import json
+import sqlite3
+from pathlib import Path
+from typing import Any
+
+from pyagent_context.item import ContextItem
+
+
+class SessionMemory:
+    """Session-scoped memory that persists across turns.
+
+    Supports two backends:
+    - ``json``: simple JSON file per session (default)
+    - ``sqlite``: SQLite database for concurrent-safe access
+
+    Args:
+        session_id: Unique session identifier.
+        backend: ``"json"`` or ``"sqlite"``.
+        storage_path: Directory for storage files. Defaults to ``".pyagent_sessions"``.
+    """
+
+    def __init__(
+        self,
+        session_id: str,
+        backend: str = "json",
+        storage_path: str | Path = ".pyagent_sessions",
+    ) -> None:
+        self._session_id = session_id
+        self._backend = backend
+        self._storage_path = Path(storage_path)
+        self._storage_path.mkdir(parents=True, exist_ok=True)
+        self._items: list[ContextItem] = []
+        self._loaded = False
+
+    @property
+    def session_id(self) -> str:
+        return self._session_id
+
+    def add(self, item: ContextItem) -> None:
+        """Add an item to the session."""
+        self._ensure_loaded()
+        self._items.append(item)
+
+    def get_all(self) -> list[ContextItem]:
+        """Return all items in this session."""
+        self._ensure_loaded()
+        return list(self._items)
+
+    def save(self) -> None:
+        """Persist current items to storage."""
+        if self._backend == "sqlite":
+            self._save_sqlite()
+        else:
+            self._save_json()
+
+    def load(self) -> None:
+        """Load items from storage."""
+        if self._backend == "sqlite":
+            self._load_sqlite()
+        else:
+            self._load_json()
+        self._loaded = True
+
+    def clear(self) -> None:
+        """Remove all items and delete persisted data."""
+        self._items.clear()
+        path = self._file_path
+        if path.exists():
+            path.unlink()
+
+    # ------------------------------------------------------------------
+    # JSON backend
+    # ------------------------------------------------------------------
+
+    def _save_json(self) -> None:
+        path = self._file_path
+        data = [item.to_dict() for item in self._items]
+        path.write_text(json.dumps(data, indent=2))
+
+    def _load_json(self) -> None:
+        path = self._file_path
+        if path.exists():
+            data = json.loads(path.read_text())
+            self._items = [ContextItem.from_dict(d) for d in data]
+        else:
+            self._items = []
+
+    # ------------------------------------------------------------------
+    # SQLite backend
+    # ------------------------------------------------------------------
+
+    def _save_sqlite(self) -> None:
+        db_path = self._storage_path / "sessions.db"
+        conn = sqlite3.connect(str(db_path))
+        try:
+            conn.execute(
+                "CREATE TABLE IF NOT EXISTS context_items "
+                "(session_id TEXT, item_id TEXT PRIMARY KEY, data TEXT)"
+            )
+            conn.execute(
+                "DELETE FROM context_items WHERE session_id = ?",
+                (self._session_id,),
+            )
+            for item in self._items:
+                conn.execute(
+                    "INSERT INTO context_items (session_id, item_id, data) VALUES (?, ?, ?)",
+                    (self._session_id, item.id, json.dumps(item.to_dict())),
+                )
+            conn.commit()
+        finally:
+            conn.close()
+
+    def _load_sqlite(self) -> None:
+        db_path = self._storage_path / "sessions.db"
+        if not db_path.exists():
+            self._items = []
+            return
+        conn = sqlite3.connect(str(db_path))
+        try:
+            conn.execute(
+                "CREATE TABLE IF NOT EXISTS context_items "
+                "(session_id TEXT, item_id TEXT PRIMARY KEY, data TEXT)"
+            )
+            cursor = conn.execute(
+                "SELECT data FROM context_items WHERE session_id = ? ORDER BY rowid",
+                (self._session_id,),
+            )
+            self._items = [ContextItem.from_dict(json.loads(row[0])) for row in cursor]
+        finally:
+            conn.close()
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+
+    @property
+    def _file_path(self) -> Path:
+        suffix = ".db" if self._backend == "sqlite" else ".json"
+        return self._storage_path / f"{self._session_id}{suffix}"
+
+    def _ensure_loaded(self) -> None:
+        if not self._loaded:
+            self.load()
+
+    def __len__(self) -> int:
+        self._ensure_loaded()
+        return len(self._items)

pyagent_context/memory/working.py

@@ -0,0 +1,69 @@
+"""WorkingMemory: bounded deque with automatic eviction and token limit."""
+
+from __future__ import annotations
+
+from collections import deque
+
+from pyagent_context.item import ContextItem
+
+
+class WorkingMemory:
+    """Bounded working memory for a single pattern run.
+
+    Evicts the oldest items when capacity is exceeded (either by count
+    or by total token estimate).
+
+    Args:
+        max_items: Maximum number of items to keep.
+        max_tokens: Maximum total token estimate before eviction.
+    """
+
+    def __init__(
+        self,
+        max_items: int = 100,
+        max_tokens: int = 50_000,
+    ) -> None:
+        self._max_items = max_items
+        self._max_tokens = max_tokens
+        self._items: deque[ContextItem] = deque()
+
+    def add(self, item: ContextItem) -> list[ContextItem]:
+        """Add an item, evicting oldest if necessary.
+
+        Returns:
+            List of evicted items (empty if none were evicted).
+        """
+        evicted: list[ContextItem] = []
+        self._items.append(item)
+
+        # Evict by count
+        while len(self._items) > self._max_items:
+            evicted.append(self._items.popleft())
+
+        # Evict by token budget
+        while self.total_tokens > self._max_tokens and len(self._items) > 1:
+            evicted.append(self._items.popleft())
+
+        return evicted
+
+    @property
+    def total_tokens(self) -> int:
+        return sum(item.token_estimate for item in self._items)
+
+    @property
+    def items(self) -> list[ContextItem]:
+        return list(self._items)
+
+    @property
+    def utilization(self) -> float:
+        """Token utilization as a fraction of max_tokens."""
+        return self.total_tokens / self._max_tokens if self._max_tokens > 0 else 0.0
+
+    def clear(self) -> None:
+        self._items.clear()
+
+    def __len__(self) -> int:
+        return len(self._items)
+
+    def __bool__(self) -> bool:
+        return len(self._items) > 0

pyagent_context/redaction.py

@@ -0,0 +1,71 @@
+"""ContextRedactor: field-level redaction by sensitivity tier."""
+
+from __future__ import annotations
+
+from pyagent_context.item import ContextItem, Sensitivity, SENSITIVITY_ORDER
+from pyagent_context.ledger import ContextLedger
+
+
+class ContextRedactor:
+    """Redact or filter context items based on sensitivity.
+
+    Items above the allowed sensitivity threshold are either fully redacted
+    (content replaced) or excluded entirely.
+
+    Args:
+        max_sensitivity: Maximum allowed sensitivity level. Items above
+            this are redacted.
+        redaction_text: Replacement text for redacted items.
+        exclude_above: If ``True``, items above max_sensitivity are
+            excluded instead of redacted.
+    """
+
+    def __init__(
+        self,
+        max_sensitivity: Sensitivity = Sensitivity.INTERNAL,
+        redaction_text: str = "[REDACTED]",
+        exclude_above: bool = False,
+    ) -> None:
+        self._max_sensitivity = max_sensitivity
+        self._redaction_text = redaction_text
+        self._exclude_above = exclude_above
+
+    def redact_item(self, item: ContextItem) -> ContextItem | None:
+        """Redact a single item if it exceeds the sensitivity threshold.
+
+        Returns:
+            The original item if within threshold, a redacted copy if above,
+            or ``None`` if ``exclude_above`` is ``True`` and the item exceeds.
+        """
+        item_level = SENSITIVITY_ORDER.get(item.sensitivity, 0)
+        max_level = SENSITIVITY_ORDER.get(self._max_sensitivity, 0)
+
+        if item_level <= max_level:
+            return item
+
+        if self._exclude_above:
+            return None
+
+        return ContextItem(
+            content=self._redaction_text,
+            source=item.source,
+            timestamp=item.timestamp,
+            trust_level=item.trust_level,
+            sensitivity=item.sensitivity,
+            expires_at=item.expires_at,
+            derived_from=item.id,
+            token_estimate=max(1, len(self._redaction_text) // 4),
+        )
+
+    def redact_ledger(self, ledger: ContextLedger) -> ContextLedger:
+        """Apply redaction to all items in a ledger.
+
+        Returns:
+            New ledger with redacted/filtered items.
+        """
+        new_items: list[ContextItem] = []
+        for item in ledger.items:
+            result = self.redact_item(item)
+            if result is not None:
+                new_items.append(result)
+        return ContextLedger(items=new_items)

pyagent_context/retrieval.py

@@ -0,0 +1,121 @@
+"""TrustAwareRetriever: score candidates by trust × recency × relevance."""
+
+from __future__ import annotations
+
+import math
+import time
+from dataclasses import dataclass
+
+from pyagent_context.item import ContextItem, TRUST_ORDER
+from pyagent_context.ledger import ContextLedger
+
+
+@dataclass(frozen=True)
+class ScoredItem:
+    """A context item with a composite retrieval score.
+
+    Attributes:
+        item: The context item.
+        score: Composite score (0.0–1.0).
+        trust_score: Trust component of the score.
+        recency_score: Recency component of the score.
+        relevance_score: Relevance component of the score.
+    """
+
+    item: ContextItem
+    score: float
+    trust_score: float
+    recency_score: float
+    relevance_score: float
+
+
+class TrustAwareRetriever:
+    """Retrieve context items scored by trust, recency, and keyword relevance.
+
+    Scoring formula:
+        ``score = w_trust * trust + w_recency * recency + w_relevance * relevance``
+
+    Where:
+    - **trust** = ``TRUST_ORDER[item.trust_level] / 3.0``
+    - **recency** = ``exp(-age_seconds / half_life)``
+    - **relevance** = keyword overlap ratio with query
+
+    Args:
+        weight_trust: Weight for trust component.
+        weight_recency: Weight for recency component.
+        weight_relevance: Weight for relevance component.
+        recency_half_life: Seconds for recency score to halve.
+    """
+
+    def __init__(
+        self,
+        weight_trust: float = 0.3,
+        weight_recency: float = 0.3,
+        weight_relevance: float = 0.4,
+        recency_half_life: float = 3600.0,
+    ) -> None:
+        self._w_trust = weight_trust
+        self._w_recency = weight_recency
+        self._w_relevance = weight_relevance
+        self._half_life = recency_half_life
+
+    def retrieve(
+        self,
+        ledger: ContextLedger,
+        query: str,
+        *,
+        top_k: int = 10,
+        min_score: float = 0.0,
+    ) -> list[ScoredItem]:
+        """Score and rank all items in the ledger against a query.
+
+        Args:
+            ledger: The context ledger to search.
+            query: The query string for relevance scoring.
+            top_k: Maximum results to return.
+            min_score: Minimum composite score to include.
+
+        Returns:
+            Ranked list of ``ScoredItem`` objects.
+        """
+        now = time.time()
+        query_words = set(query.lower().split())
+        results: list[ScoredItem] = []
+
+        for item in ledger.items:
+            if item.is_expired:
+                continue
+
+            trust = TRUST_ORDER.get(item.trust_level, 0) / 3.0
+            age = now - item.timestamp
+            recency = math.exp(-age / self._half_life) if self._half_life > 0 else 0.0
+            relevance = self._keyword_relevance(item.content, query_words)
+
+            score = (
+                self._w_trust * trust
+                + self._w_recency * recency
+                + self._w_relevance * relevance
+            )
+
+            if score >= min_score:
+                results.append(
+                    ScoredItem(
+                        item=item,
+                        score=score,
+                        trust_score=trust,
+                        recency_score=recency,
+                        relevance_score=relevance,
+                    )
+                )
+
+        results.sort(key=lambda r: r.score, reverse=True)
+        return results[:top_k]
+
+    @staticmethod
+    def _keyword_relevance(content: str, query_words: set[str]) -> float:
+        """Simple keyword overlap ratio."""
+        if not query_words:
+            return 0.0
+        content_words = set(content.lower().split())
+        overlap = query_words & content_words
+        return len(overlap) / len(query_words) if query_words else 0.0