patternmem-rag 0.1.1__py3-none-any.whl

patternmem/backends/neo4j_backend.py

@@ -0,0 +1,236 @@
+"""
+patternmem.backends.neo4j_backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+Neo4j / AuraDB backend for PatternMem.  Requires the ``[neo4j]`` optional extra::
+
+    pip install patternmem-rag[neo4j]
+
+Graph schema (implement exactly — do not deviate)
+--------------------------------------------------
+Nodes
+~~~~~
+- ``FailurePattern {id, query_embedding, failure_type, root_cause, hint_text,
+                    score, created_at, decay_weight}``
+- ``QueryType {label}``   — values: temporal | entity | multi-hop | factoid
+- ``RetrievalHint {text}``
+
+Edges
+~~~~~
+- ``(:FailurePattern)-[:TRIGGERED_BY]->(:QueryType)``
+- ``(:FailurePattern)-[:SUGGESTS]->(:RetrievalHint)``
+- ``(:RetrievalHint)-[:SIMILAR_TO {sim_score}]->(:RetrievalHint)``
+
+Vector index
+~~~~~~~~~~~~
+Named ``'failure_pattern_embeddings'`` on the ``query_embedding`` property of
+``FailurePattern`` nodes.  Created automatically on first use.
+
+Lookup Cypher (from spec — do not modify)
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+::
+
+    CALL db.index.vector.queryNodes('failure_pattern_embeddings', 5, $query_embedding)
+    YIELD node, score
+    WHERE score > 0.82
+    RETURN node.hint_text, node.failure_type
+    ORDER BY score DESC LIMIT 3
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from typing import Any
+
+try:
+    from neo4j import AsyncGraphDatabase  # type: ignore[import]
+    from neo4j.exceptions import ClientError  # type: ignore[import]
+
+    _NEO4J_AVAILABLE = True
+except ImportError:
+    _NEO4J_AVAILABLE = False
+
+from patternmem.backend import MemoryBackend
+from patternmem.types import FailurePattern, FailureType
+
+# ---------------------------------------------------------------------------
+# Cypher statements
+# ---------------------------------------------------------------------------
+
+_CREATE_VECTOR_INDEX = """
+CREATE VECTOR INDEX failure_pattern_embeddings IF NOT EXISTS
+FOR (n:FailurePattern)
+ON n.query_embedding
+OPTIONS {indexConfig: {`vector.dimensions`: 384, `vector.similarity_function`: 'cosine'}}
+"""
+
+_UPSERT_PATTERN = """
+MERGE (p:FailurePattern {id: $id})
+SET p.query_embedding = $query_embedding,
+    p.failure_type    = $failure_type,
+    p.root_cause      = $root_cause,
+    p.hint_text       = $hint_text,
+    p.score           = $score,
+    p.created_at      = $created_at,
+    p.decay_weight    = $decay_weight
+WITH p
+MERGE (h:RetrievalHint {text: $hint_text})
+MERGE (p)-[:SUGGESTS]->(h)
+"""
+
+_LOOKUP_PATTERNS = """
+CALL db.index.vector.queryNodes('failure_pattern_embeddings', 5, $query_embedding)
+YIELD node, score
+WHERE score > $threshold
+RETURN node.id           AS id,
+       node.query_embedding AS query_embedding,
+       node.failure_type AS failure_type,
+       node.root_cause   AS root_cause,
+       node.hint_text    AS hint_text,
+       node.score        AS score,
+       node.created_at   AS created_at,
+       node.decay_weight AS decay_weight,
+       score             AS sim_score
+ORDER BY score DESC LIMIT 3
+"""
+
+_COUNT_PATTERNS = "MATCH (p:FailurePattern) RETURN count(p) AS cnt"
+
+_UPDATE_DECAY = """
+MATCH (p:FailurePattern {id: $id})
+SET p.decay_weight = $decay_weight
+RETURN p.id AS id
+"""
+
+_DELETE_PATTERN = """
+MATCH (p:FailurePattern {id: $id})
+DETACH DELETE p
+RETURN $id AS id
+"""
+
+
+class Neo4jBackend(MemoryBackend):
+    """Neo4j / AuraDB backend with native vector index lookup.
+
+    Parameters
+    ----------
+    uri:
+        Bolt or neo4j+s URI (e.g. ``"bolt://localhost:7687"`` or
+        ``"neo4j+s://<aura-id>.databases.neo4j.io"``).
+    auth:
+        ``(username, password)`` tuple.
+    database:
+        Target database name.  ``"neo4j"`` for most single-DB setups.
+    similarity_threshold:
+        Cosine similarity floor for the vector index lookup.  Passed as
+        ``$threshold`` to the lookup Cypher.
+    """
+
+    def __init__(
+        self,
+        uri: str,
+        auth: tuple[str, str],
+        database: str = "neo4j",
+        similarity_threshold: float = 0.82,
+    ) -> None:
+        if not _NEO4J_AVAILABLE:
+            raise ImportError(
+                "Neo4jBackend requires the 'neo4j' driver. "
+                "Install it with: pip install patternmem-rag[neo4j]"
+            )
+        self._driver = AsyncGraphDatabase.driver(uri, auth=auth)
+        self._database = database
+        self._threshold = similarity_threshold
+        self._index_ready = False
+
+    async def _ensure_index(self) -> None:
+        """Create the vector index if it does not already exist."""
+        if self._index_ready:
+            return
+        async with self._driver.session(database=self._database) as session:
+            try:
+                await session.run(_CREATE_VECTOR_INDEX)
+            except ClientError:
+                # Index may already exist under a different schema — ignore
+                pass
+        self._index_ready = True
+
+    async def write_pattern(self, pattern: FailurePattern) -> None:
+        await self._ensure_index()
+        async with self._driver.session(database=self._database) as session:
+            await session.run(
+                _UPSERT_PATTERN,
+                id=pattern.id,
+                query_embedding=pattern.query_embedding,
+                failure_type=pattern.failure_type.name,
+                root_cause=pattern.root_cause,
+                hint_text=pattern.hint_text,
+                score=pattern.score,
+                created_at=pattern.created_at.isoformat(),
+                decay_weight=pattern.decay_weight,
+            )
+
+    async def lookup_patterns(
+        self,
+        query_embedding: list[float],
+        top_k: int = 3,
+    ) -> list[FailurePattern]:
+        await self._ensure_index()
+        async with self._driver.session(database=self._database) as session:
+            result = await session.run(
+                _LOOKUP_PATTERNS,
+                query_embedding=query_embedding,
+                threshold=self._threshold,
+            )
+            records = await result.data()
+
+        patterns: list[FailurePattern] = []
+        for rec in records:
+            patterns.append(
+                FailurePattern(
+                    id=rec["id"],
+                    query_embedding=list(rec["query_embedding"]),
+                    failure_type=FailureType[rec["failure_type"]],
+                    root_cause=rec["root_cause"],
+                    hint_text=rec["hint_text"],
+                    score=rec["score"],
+                    created_at=datetime.fromisoformat(rec["created_at"]).replace(
+                        tzinfo=timezone.utc
+                    ),
+                    decay_weight=rec["decay_weight"],
+                )
+            )
+        return patterns[:top_k]
+
+    async def get_stats(self) -> dict[str, Any]:
+        async with self._driver.session(database=self._database) as session:
+            result = await session.run(_COUNT_PATTERNS)
+            record = await result.single()
+        return {
+            "count": int(record["cnt"]) if record else 0,
+            "database": self._database,
+            "backend": "neo4j",
+        }
+
+    async def update_pattern(self, pattern_id: str, decay_weight: float) -> None:
+        async with self._driver.session(database=self._database) as session:
+            result = await session.run(
+                _UPDATE_DECAY, id=pattern_id, decay_weight=decay_weight
+            )
+            record = await result.single()
+            if record is None:
+                raise KeyError(
+                    f"Pattern {pattern_id!r} not found in Neo4j backend"
+                )
+
+    async def delete_pattern(self, pattern_id: str) -> None:
+        async with self._driver.session(database=self._database) as session:
+            result = await session.run(_DELETE_PATTERN, id=pattern_id)
+            summary = await result.consume()
+            if summary.counters.nodes_deleted == 0:
+                raise KeyError(
+                    f"Pattern {pattern_id!r} not found in Neo4j backend"
+                )
+
+    async def close(self) -> None:
+        """Close the driver connection pool.  Call on teardown."""
+        await self._driver.close()

patternmem/backends/networkx_backend.py

@@ -0,0 +1,157 @@
+"""
+patternmem.backends.networkx_backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+In-memory NetworkX DiGraph backend for PatternMem.
+
+Ideal for experimentation, notebooks, and unit tests that need a
+zero-file-I/O graph-native store.  Optionally serialises to GraphML
+via ``persist_path`` for lightweight persistence across restarts.
+
+Similarity
+----------
+Cosine similarity via numpy — identical algorithm to ``JSONBackend``.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import numpy as np
+
+try:
+    import networkx as nx
+
+    _NX_AVAILABLE = True
+except ImportError:
+    _NX_AVAILABLE = False
+
+from patternmem._utils import cosine_similarity as _cosine_similarity
+from patternmem.backend import MemoryBackend
+from patternmem.types import FailurePattern, FailureType
+
+
+
+def _pattern_to_attrs(p: FailurePattern) -> dict[str, Any]:
+    """Flatten a FailurePattern into a NetworkX node-attribute dict."""
+    return {
+        "query_embedding": p.query_embedding,
+        "failure_type": p.failure_type.name,
+        "root_cause": p.root_cause,
+        "hint_text": p.hint_text,
+        "score": p.score,
+        "created_at": p.created_at.isoformat(),
+        "decay_weight": p.decay_weight,
+    }
+
+
+def _attrs_to_pattern(node_id: str, attrs: dict[str, Any]) -> FailurePattern:
+    return FailurePattern(
+        id=node_id,
+        query_embedding=attrs["query_embedding"],
+        failure_type=FailureType[attrs["failure_type"]],
+        root_cause=attrs["root_cause"],
+        hint_text=attrs["hint_text"],
+        score=attrs["score"],
+        created_at=datetime.fromisoformat(attrs["created_at"]).replace(
+            tzinfo=timezone.utc
+        ),
+        decay_weight=attrs["decay_weight"],
+    )
+
+
+class NetworkXBackend(MemoryBackend):
+    """In-memory graph backend backed by a ``networkx.DiGraph``.
+
+    Parameters
+    ----------
+    persist_path:
+        Optional path to a GraphML file.  If provided the graph is loaded on
+        construction and saved on every write/update/delete.  ``None`` means
+        pure in-memory (no persistence).
+    similarity_threshold:
+        Minimum cosine similarity for lookup.
+    """
+
+    def __init__(
+        self,
+        persist_path: str | Path | None = None,
+        similarity_threshold: float = 0.82,
+    ) -> None:
+        if not _NX_AVAILABLE:
+            raise ImportError(
+                "NetworkXBackend requires 'networkx'. "
+                "Install it with: pip install patternmem-rag[networkx]"
+            )
+        self._threshold = similarity_threshold
+        self._persist_path = Path(persist_path) if persist_path else None
+        self._lock = asyncio.Lock()
+
+        self._graph: Any = nx.DiGraph()
+        if self._persist_path and self._persist_path.exists():
+            self._graph = nx.read_graphml(str(self._persist_path))
+            # GraphML deserialises embeddings as strings; re-parse them
+            for nid in list(self._graph.nodes):
+                raw_emb = self._graph.nodes[nid].get("query_embedding", "[]")
+                if isinstance(raw_emb, str):
+                    import json
+                    self._graph.nodes[nid]["query_embedding"] = json.loads(raw_emb)
+
+    def _persist(self) -> None:
+        if self._persist_path:
+            import json
+            # GraphML can't store lists natively — serialise embeddings to JSON str
+            tmp_graph: Any = self._graph.copy()
+            for nid in tmp_graph.nodes:
+                emb = tmp_graph.nodes[nid].get("query_embedding", [])
+                tmp_graph.nodes[nid]["query_embedding"] = json.dumps(emb)
+            self._persist_path.parent.mkdir(parents=True, exist_ok=True)
+            nx.write_graphml(tmp_graph, str(self._persist_path))
+
+    async def write_pattern(self, pattern: FailurePattern) -> None:
+        async with self._lock:
+            self._graph.add_node(pattern.id, **_pattern_to_attrs(pattern))
+            self._persist()
+
+    async def lookup_patterns(
+        self,
+        query_embedding: list[float],
+        top_k: int = 3,
+    ) -> list[FailurePattern]:
+        async with self._lock:
+            scored: list[tuple[float, FailurePattern]] = []
+            for nid, attrs in self._graph.nodes(data=True):
+                stored_emb: list[float] = attrs.get("query_embedding", [])
+                if not stored_emb:
+                    continue
+                sim = _cosine_similarity(query_embedding, stored_emb)
+                if sim >= self._threshold:
+                    scored.append((sim, _attrs_to_pattern(nid, attrs)))
+
+        scored.sort(key=lambda t: t[0], reverse=True)
+        return [p for _, p in scored[:top_k]]
+
+    async def get_stats(self) -> dict[str, Any]:
+        async with self._lock:
+            count = self._graph.number_of_nodes()
+        return {
+            "count": count,
+            "persist_path": str(self._persist_path) if self._persist_path else None,
+            "backend": "networkx",
+        }
+
+    async def update_pattern(self, pattern_id: str, decay_weight: float) -> None:
+        async with self._lock:
+            if pattern_id not in self._graph:
+                raise KeyError(f"Pattern {pattern_id!r} not found in NetworkX backend")
+            self._graph.nodes[pattern_id]["decay_weight"] = decay_weight
+            self._persist()
+
+    async def delete_pattern(self, pattern_id: str) -> None:
+        async with self._lock:
+            if pattern_id not in self._graph:
+                raise KeyError(f"Pattern {pattern_id!r} not found in NetworkX backend")
+            self._graph.remove_node(pattern_id)
+            self._persist()

patternmem/backends/sqlite_backend.py

@@ -0,0 +1,197 @@
+"""
+patternmem.backends.sqlite_backend
+~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
+SQLite backend for PatternMem using aiosqlite for async I/O.
+
+Storage layout
+--------------
+A single SQLite database file (WAL mode) with a ``failure_patterns`` table.
+The query embedding is stored as a JSON text column — no extension required.
+
+Similarity
+----------
+All embeddings are loaded into numpy for brute-force cosine similarity.
+This is acceptable for typical pattern-memory sizes (< 10 000 patterns).
+For larger workloads, use Neo4j with its native vector index.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any
+
+import json
+import numpy as np
+import aiosqlite
+
+from patternmem._utils import cosine_similarity as _cosine_similarity
+from patternmem.backend import MemoryBackend
+from patternmem.types import FailurePattern, FailureType
+
+_DEFAULT_PATH = Path.home() / ".patternmem" / "patterns.db"
+
+_CREATE_TABLE = """
+CREATE TABLE IF NOT EXISTS failure_patterns (
+    id           TEXT PRIMARY KEY,
+    query_embedding TEXT NOT NULL,          -- JSON array of floats
+    failure_type TEXT NOT NULL,
+    root_cause   TEXT NOT NULL DEFAULT '',
+    hint_text    TEXT NOT NULL DEFAULT '',
+    score        REAL NOT NULL DEFAULT 0.0,
+    created_at   TEXT NOT NULL,             -- ISO-8601 UTC
+    decay_weight REAL NOT NULL DEFAULT 1.0
+)
+"""
+
+
+
+def _row_to_pattern(row: aiosqlite.Row) -> FailurePattern:
+    return FailurePattern(
+        id=row["id"],
+        query_embedding=json.loads(row["query_embedding"]),
+        failure_type=FailureType[row["failure_type"]],
+        root_cause=row["root_cause"],
+        hint_text=row["hint_text"],
+        score=row["score"],
+        created_at=datetime.fromisoformat(row["created_at"]).replace(tzinfo=timezone.utc),
+        decay_weight=row["decay_weight"],
+    )
+
+
+class SQLiteBackend(MemoryBackend):
+    """Async SQLite backend via ``aiosqlite``.
+
+    Each public method opens a fresh ``aiosqlite`` connection using
+    ``aiosqlite.connect()`` as an async context manager, performs its
+    operation, and lets the context manager close it cleanly.  This is the
+    correct aiosqlite usage pattern and avoids the "thread can only be started
+    once" error that arises from reusing a connection object.
+
+    Parameters
+    ----------
+    path:
+        Path to the SQLite file.  Created automatically if absent.
+    similarity_threshold:
+        Minimum cosine similarity for lookup.
+    """
+
+    def __init__(
+        self,
+        path: str | Path = _DEFAULT_PATH,
+        similarity_threshold: float = 0.82,
+    ) -> None:
+        self._path = Path(path)
+        self._threshold = similarity_threshold
+
+    def _ensure_dir(self) -> None:
+        self._path.parent.mkdir(parents=True, exist_ok=True)
+
+    async def _init_schema(self, conn: aiosqlite.Connection) -> None:
+        """Enable WAL and create the table if needed."""
+        await conn.execute("PRAGMA journal_mode=WAL")
+        await conn.execute(_CREATE_TABLE)
+        await conn.commit()
+
+    async def write_pattern(self, pattern: FailurePattern) -> None:
+        self._ensure_dir()
+        async with aiosqlite.connect(str(self._path)) as conn:
+            conn.row_factory = aiosqlite.Row
+            await self._init_schema(conn)
+            await conn.execute(
+                """
+                INSERT INTO failure_patterns
+                    (id, query_embedding, failure_type, root_cause,
+                     hint_text, score, created_at, decay_weight)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                ON CONFLICT(id) DO UPDATE SET
+                    query_embedding = excluded.query_embedding,
+                    failure_type    = excluded.failure_type,
+                    root_cause      = excluded.root_cause,
+                    hint_text       = excluded.hint_text,
+                    score           = excluded.score,
+                    created_at      = excluded.created_at,
+                    decay_weight    = excluded.decay_weight
+                """,
+                (
+                    pattern.id,
+                    json.dumps(pattern.query_embedding),
+                    pattern.failure_type.name,
+                    pattern.root_cause,
+                    pattern.hint_text,
+                    pattern.score,
+                    pattern.created_at.isoformat(),
+                    pattern.decay_weight,
+                ),
+            )
+            await conn.commit()
+
+    async def lookup_patterns(
+        self,
+        query_embedding: list[float],
+        top_k: int = 3,
+    ) -> list[FailurePattern]:
+        self._ensure_dir()
+        async with aiosqlite.connect(str(self._path)) as conn:
+            conn.row_factory = aiosqlite.Row
+            await self._init_schema(conn)
+            async with conn.execute(
+                "SELECT * FROM failure_patterns"
+            ) as cursor:
+                rows = await cursor.fetchall()
+
+        if not rows:
+            return []
+
+        scored: list[tuple[float, FailurePattern]] = []
+        for row in rows:
+            stored_emb: list[float] = json.loads(row["query_embedding"])
+            if not stored_emb:
+                continue
+            sim = _cosine_similarity(query_embedding, stored_emb)
+            if sim >= self._threshold:
+                scored.append((sim, _row_to_pattern(row)))
+
+        scored.sort(key=lambda t: t[0], reverse=True)
+        return [p for _, p in scored[:top_k]]
+
+    async def get_stats(self) -> dict[str, Any]:
+        self._ensure_dir()
+        async with aiosqlite.connect(str(self._path)) as conn:
+            conn.row_factory = aiosqlite.Row
+            await self._init_schema(conn)
+            async with conn.execute(
+                "SELECT COUNT(*) as cnt FROM failure_patterns"
+            ) as cursor:
+                row = await cursor.fetchone()
+        return {
+            "count": int(row["cnt"]) if row else 0,
+            "path": str(self._path),
+            "backend": "sqlite",
+        }
+
+    async def update_pattern(self, pattern_id: str, decay_weight: float) -> None:
+        self._ensure_dir()
+        async with aiosqlite.connect(str(self._path)) as conn:
+            conn.row_factory = aiosqlite.Row
+            await self._init_schema(conn)
+            result = await conn.execute(
+                "UPDATE failure_patterns SET decay_weight = ? WHERE id = ?",
+                (decay_weight, pattern_id),
+            )
+            await conn.commit()
+            if result.rowcount == 0:
+                raise KeyError(f"Pattern {pattern_id!r} not found in SQLite backend")
+
+    async def delete_pattern(self, pattern_id: str) -> None:
+        self._ensure_dir()
+        async with aiosqlite.connect(str(self._path)) as conn:
+            conn.row_factory = aiosqlite.Row
+            await self._init_schema(conn)
+            result = await conn.execute(
+                "DELETE FROM failure_patterns WHERE id = ?",
+                (pattern_id,),
+            )
+            await conn.commit()
+            if result.rowcount == 0:
+                raise KeyError(f"Pattern {pattern_id!r} not found in SQLite backend")

patternmem/decay.py

@@ -0,0 +1,90 @@
+"""
+patternmem.decay
+~~~~~~~~~~~~~~~~
+Pure decay/eviction functions for PatternMem pattern memory.
+
+These are *pure functions* — no I/O, no async, no side effects.  They are
+the only place in the codebase that contains the numeric constants that govern
+pattern reinforcement and eviction.  Tests may monkey-patch these constants.
+
+Constants
+---------
+REINFORCE_DELTA : float
+    Amount added to ``decay_weight`` when a hint-augmented run improves the
+    score by more than ``IMPROVEMENT_GAP``.
+DECAY_DELTA : float
+    Amount subtracted from ``decay_weight`` when a hint-augmented run does *not*
+    improve the score by the required gap.
+EVICTION_FLOOR : float
+    Patterns whose ``decay_weight`` falls below this value are deleted.
+IMPROVEMENT_GAP : float
+    Minimum score improvement required to be counted as a reinforcement event.
+"""
+
+from __future__ import annotations
+
+# ---------------------------------------------------------------------------
+# Module-level constants (single source of truth for decay numerics)
+# ---------------------------------------------------------------------------
+
+REINFORCE_DELTA: float = 0.1
+DECAY_DELTA: float = 0.2
+EVICTION_FLOOR: float = 0.1
+IMPROVEMENT_GAP: float = 0.15
+
+
+# ---------------------------------------------------------------------------
+# Public functions
+# ---------------------------------------------------------------------------
+
+
+def compute_new_decay_weight(
+    stored: float,
+    old_score: float,
+    new_score: float,
+) -> float:
+    """Return the updated ``decay_weight`` after comparing *old_score* to *new_score*.
+
+    Parameters
+    ----------
+    stored:
+        Current ``decay_weight`` of the pattern in the backend.
+    old_score:
+        The evaluation score stored in the pattern at write time.
+    new_score:
+        The evaluation score from the most recent Phase 2 run (after hint injection).
+
+    Returns
+    -------
+    float
+        New decay weight.  May be below ``EVICTION_FLOOR`` — the caller is
+        responsible for checking ``should_evict`` and triggering deletion.
+
+    Notes
+    -----
+    The formula mirrors the spec exactly::
+
+        if new_score > stored_pattern.score + 0.15:
+            decay_weight += 0.1   # reinforce
+        else:
+            decay_weight -= 0.2   # decay
+    """
+    if new_score > old_score + IMPROVEMENT_GAP:
+        return stored + REINFORCE_DELTA
+    return stored - DECAY_DELTA
+
+
+def should_evict(decay_weight: float) -> bool:
+    """Return ``True`` if the pattern should be permanently evicted.
+
+    Parameters
+    ----------
+    decay_weight:
+        The *updated* decay weight (output of ``compute_new_decay_weight``).
+
+    Returns
+    -------
+    bool
+        ``True`` iff ``decay_weight < EVICTION_FLOOR``.
+    """
+    return decay_weight < EVICTION_FLOOR