memuron 0.1.1__py3-none-any.whl

memuron/persistence/store_helpers.py

@@ -0,0 +1,16 @@
+"""Store helpers that stay in userland (no engine fork required)."""
+
+from __future__ import annotations
+
+from artha_engine.store.projection_sql import sql_store_execute, sql_store_has_tables
+
+
+def purge_arthaanu(store: object, artha_id: str) -> bool:
+    """Remove the current Arthaanu projection row without appending a store.delete event."""
+    if sql_store_has_tables(store):
+        sql_store_execute(store, "DELETE FROM arthaanu WHERE artha_id = ?", (artha_id,))
+        return True
+    items = getattr(store, "_items", None)
+    if isinstance(items, dict):
+        return items.pop(artha_id, None) is not None
+    return False

memuron/search/__init__.py

@@ -0,0 +1 @@
+"""Vector, full-text, and hybrid retrieval backends."""

memuron/search/fulltext.py

@@ -0,0 +1,110 @@
+"""PostgreSQL full-text (tsvector + GIN) for hybrid keyword retrieval."""
+
+from __future__ import annotations
+
+import logging
+from typing import Any
+
+from artha_engine.store.projection_sql import sql_store_execute, sql_store_fetchall, sql_store_has_tables
+
+from memuron.domain.scope_filter import scope_sql_clauses
+
+logger = logging.getLogger(__name__)
+
+_FULLTEXT_FLAG = "_memuron_fulltext_ready"
+
+
+def fulltext_is_ready(store: object) -> bool:
+    return bool(getattr(store, _FULLTEXT_FLAG, False))
+
+
+def ensure_fulltext_schema(store: object) -> bool:
+    """Add content_tsv column, backfill, and GIN index for BM25-style retrieval."""
+    if not sql_store_has_tables(store):
+        return False
+    if fulltext_is_ready(store):
+        return True
+    try:
+        sql_store_execute(
+            store,
+            "ALTER TABLE memuron_memories ADD COLUMN IF NOT EXISTS content_tsv tsvector",
+        )
+        sql_store_execute(
+            store,
+            """
+            UPDATE memuron_memories
+            SET content_tsv = to_tsvector('english', coalesce(content, ''))
+            WHERE content_tsv IS NULL
+            """,
+        )
+        sql_store_execute(
+            store,
+            """
+            CREATE INDEX IF NOT EXISTS idx_memuron_memories_content_tsv_gin
+            ON memuron_memories USING GIN (content_tsv)
+            """,
+        )
+        setattr(store, _FULLTEXT_FLAG, True)
+        logger.info("memuron full-text (tsvector GIN) ready")
+        return True
+    except Exception as exc:
+        logger.warning("full-text setup skipped: %s", exc)
+        return False
+
+
+def sync_memory_fulltext(store: object, artha_id: str, content: str) -> None:
+    if not fulltext_is_ready(store):
+        return
+    sql_store_execute(
+        store,
+        """
+        UPDATE memuron_memories
+        SET content_tsv = to_tsvector('english', coalesce(?, ''))
+        WHERE artha_id = ?
+        """,
+        (content, artha_id),
+    )
+
+
+def _scope_where(scope: list[str] | None) -> tuple[str, list[str]]:
+    scope_sql, scope_params = scope_sql_clauses("scope_json", scope)
+    if scope_sql:
+        return f" AND {scope_sql}", scope_params
+    return "", []
+
+
+def fulltext_memory_search(
+    store: object,
+    query: str,
+    *,
+    top_k: int,
+    scope: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    """Rank memories with Postgres ts_rank_cd over content_tsv."""
+    if not fulltext_is_ready(store) or not query.strip():
+        return []
+
+    scope_where, scope_params = _scope_where(scope)
+    rows = sql_store_fetchall(
+        store,
+        f"""
+        SELECT artha_id, content, scope_json, metadata_json,
+               ts_rank_cd(content_tsv, plainto_tsquery('english', ?)) AS fts_score
+        FROM memuron_memories
+        WHERE content_tsv IS NOT NULL
+          AND content_tsv @@ plainto_tsquery('english', ?){scope_where}
+        ORDER BY fts_score DESC
+        LIMIT ?
+        """,
+        (query, query, *scope_params, top_k),
+    )
+    return [
+        {
+            "artha_id": str(row["artha_id"]),
+            "content": str(row.get("content") or ""),
+            "scope_json": row.get("scope_json"),
+            "metadata_json": row.get("metadata_json"),
+            "fts_score": float(row["fts_score"]),
+        }
+        for row in rows
+    ]

memuron/search/hybrid.py

@@ -0,0 +1,284 @@
+"""Hybrid memory retrieval: pgvector + Postgres FTS fused with RRF."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from artha_engine import ArthaEngine, TextArthaanu
+from artha_engine.decoders.search import Bm25SearchDecoder, Bm25SearchParams, TextCandidate
+from artha_engine.retrieval.rrf import reciprocal_rank_fusion
+
+from memuron.application.config import settings
+from memuron.search.fulltext import fulltext_is_ready, fulltext_memory_search
+from memuron.search.pgvector import pgvector_is_ready, pgvector_memory_search
+from memuron.domain.representations import MemoryArthaanu, MemoryValue
+from memuron.domain.scope_filter import scope_matches_filter
+from memuron.domain.schemas import source_identity_from_metadata
+
+
+def retrieve_pool_size(k: int, multiplier: int | None = None) -> int:
+    mult = multiplier if multiplier is not None else settings.search_retrieve_multiplier
+    return max(k * mult, 40)
+
+
+def _memory_item(memory_id: str, content: str, scope: list[str]) -> MemoryArthaanu:
+    return MemoryArthaanu(
+        artha_id=memory_id,
+        name="memory",
+        value=MemoryValue(content=content, scope=scope, embedding=[]),
+    )
+
+
+def _bm25_hits_in_memory(
+    engine: ArthaEngine,
+    query: str,
+    corpus: list[tuple[str, str, list[str], dict[str, Any]]],
+    *,
+    top_k: int,
+) -> list[tuple[str, float]]:
+    if not corpus:
+        return []
+    decoder = Bm25SearchDecoder()
+    candidates = [
+        TextCandidate(
+            item=_memory_item(memory_id, content, scope),
+            text=TextArthaanu(name="content", value=content),
+        )
+        for memory_id, content, scope, _metadata in corpus
+    ]
+    result = decoder.decode(
+        TextArthaanu(name="query", value=query),
+        Bm25SearchParams(candidates=candidates, top_k=top_k),
+    )
+    return [(hit.item.artha_id, float(hit.score)) for hit in result.hits]
+
+
+def _sqlite_corpus(
+    engine: ArthaEngine,
+    scope: list[str] | None,
+) -> list[tuple[str, str, list[str], dict[str, Any]]]:
+    from memuron.memory.recipes import _fetch_memory_rows_by_ids, _list_memory_search_rows, _parse_json_field
+
+    rows = _list_memory_search_rows(engine)
+    if not rows:
+        return []
+    details = _fetch_memory_rows_by_ids(engine, [str(row["artha_id"]) for row in rows])
+    corpus: list[tuple[str, str, list[str], dict[str, Any]]] = []
+    for memory_id, row in details.items():
+        scope_tokens = _parse_json_field(row.get("scope_json"), [])
+        if not isinstance(scope_tokens, list):
+            scope_tokens = []
+        if scope and not scope_matches_filter([str(s) for s in scope_tokens], scope):
+            continue
+        content = str(row.get("content") or "")
+        metadata = _parse_json_field(row.get("metadata_json"), {})
+        if content.strip():
+            corpus.append(
+                (
+                    memory_id,
+                    content,
+                    [str(s) for s in scope_tokens],
+                    metadata if isinstance(metadata, dict) else {},
+                )
+            )
+    return corpus
+
+
+def _vector_memory_hits(
+    engine: ArthaEngine,
+    query_vector: list[float],
+    *,
+    top_k: int,
+    scope: list[str] | None,
+    min_semantic_score: float,
+) -> list[tuple[str, float, dict[str, Any]]]:
+    """Vector leg; min_semantic_score applies here only (not to FTS/RRF)."""
+    from memuron.memory.recipes import (
+        _fetch_memory_rows_by_ids,
+        _memory_similarity_hits,
+        _parse_json_field,
+        _row_to_memory_dict,
+    )
+
+    store = engine.store
+    output: list[tuple[str, float, dict[str, Any]]] = []
+    if pgvector_is_ready(store):
+        for row in pgvector_memory_search(
+            store,
+            query_vector,
+            top_k=top_k,
+            scope=scope,
+            include_content=True,
+        ):
+            score = float(row["semantic_score"])
+            if score < min_semantic_score:
+                continue
+            memory_id = str(row["artha_id"])
+            scope_tokens = _parse_json_field(row.get("scope_json"), [])
+            output.append(
+                (
+                    memory_id,
+                    score,
+                    {
+                        "type": "memory_node",
+                        "id": memory_id,
+                        "content": str(row.get("content") or ""),
+                        "scope": scope_tokens if isinstance(scope_tokens, list) else [],
+                        "source_metadata": source_identity_from_metadata(
+                            _parse_json_field(row.get("metadata_json"), {})
+                        ),
+                        "matched_via": "memory",
+                        "matched_via_link_id": None,
+                    },
+                )
+            )
+        return output
+
+    raw = _memory_similarity_hits(engine, query_vector, scope=scope, top_k=top_k)
+    details = _fetch_memory_rows_by_ids(engine, [memory_id for memory_id, _ in raw])
+    for memory_id, score in raw:
+        if score < min_semantic_score:
+            continue
+        row = details.get(memory_id)
+        if not row:
+            continue
+        memory = _row_to_memory_dict(row)
+        output.append(
+            (
+                memory_id,
+                score,
+                {
+                    "type": "memory_node",
+                    "id": memory_id,
+                    "content": memory["content"],
+                    "scope": memory.get("scope") or [],
+                    "source_metadata": source_identity_from_metadata(
+                        memory.get("metadata") if isinstance(memory.get("metadata"), dict) else {}
+                    ),
+                    "matched_via": "memory",
+                    "matched_via_link_id": None,
+                },
+            )
+        )
+    return output
+
+
+def _fts_memory_hits(
+    engine: ArthaEngine,
+    query: str,
+    *,
+    top_k: int,
+    scope: list[str] | None,
+) -> list[tuple[str, float, dict[str, Any]]]:
+    from memuron.memory.recipes import _parse_json_field
+
+    store = engine.store
+    if fulltext_is_ready(store):
+        output: list[tuple[str, float, dict[str, Any]]] = []
+        for row in fulltext_memory_search(store, query, top_k=top_k, scope=scope):
+            scope_tokens = _parse_json_field(row.get("scope_json"), [])
+            memory_id = str(row["artha_id"])
+            output.append(
+                (
+                    memory_id,
+                    float(row["fts_score"]),
+                    {
+                        "type": "memory_node",
+                        "id": memory_id,
+                        "content": str(row.get("content") or ""),
+                        "scope": scope_tokens if isinstance(scope_tokens, list) else [],
+                        "source_metadata": source_identity_from_metadata(
+                            _parse_json_field(row.get("metadata_json"), {})
+                        ),
+                        "matched_via": "memory",
+                        "matched_via_link_id": None,
+                    },
+                )
+            )
+        return output
+
+    corpus = _sqlite_corpus(engine, scope)
+    ranked = _bm25_hits_in_memory(engine, query, corpus, top_k=top_k)
+    by_id = {
+        memory_id: (content, scope, metadata)
+        for memory_id, content, scope, metadata in corpus
+    }
+    return [
+        (
+            memory_id,
+            score,
+            {
+                "type": "memory_node",
+                "id": memory_id,
+                "content": by_id[memory_id][0],
+                "scope": by_id[memory_id][1],
+                "source_metadata": source_identity_from_metadata(by_id[memory_id][2]),
+                "matched_via": "memory",
+                "matched_via_link_id": None,
+            },
+        )
+        for memory_id, score in ranked
+        if memory_id in by_id
+    ]
+
+
+def hybrid_memory_search(
+    engine: ArthaEngine,
+    query: str,
+    query_vector: list[float],
+    *,
+    k: int,
+    scope: list[str] | None,
+    min_semantic_score: float | None = None,
+) -> list[tuple[str, dict[str, Any], float]]:
+    """RRF-fuse vector (thresholded) + FTS/BM25 legs. Returns (kind, payload, score) tuples."""
+    min_score = (
+        min_semantic_score
+        if min_semantic_score is not None
+        else settings.search_min_semantic_score
+    )
+    pool = retrieve_pool_size(k)
+
+    vector_rows = _vector_memory_hits(
+        engine,
+        query_vector,
+        top_k=pool,
+        scope=scope,
+        min_semantic_score=min_score,
+    )
+    fts_rows = _fts_memory_hits(engine, query, top_k=pool, scope=scope)
+
+    payload_by_id: dict[str, dict[str, Any]] = {}
+    for memory_id, _score, payload in vector_rows + fts_rows:
+        payload_by_id[memory_id] = payload
+
+    vector_ranked = [memory_id for memory_id, _score, _payload in vector_rows]
+    fts_ranked = [memory_id for memory_id, _score, _payload in fts_rows]
+
+    if not vector_ranked and not fts_ranked:
+        return []
+
+    if not settings.search_hybrid or (not fts_ranked and vector_ranked):
+        # Single leg available or hybrid disabled — return vector order only.
+        return [
+            ("memory_node", payload_by_id[memory_id], score)
+            for memory_id, score, _payload in vector_rows[:k]
+        ]
+
+    if not vector_ranked:
+        return [
+            ("memory_node", payload_by_id[memory_id], score)
+            for memory_id, score, _payload in fts_rows[:k]
+        ]
+
+    fused = reciprocal_rank_fusion(
+        [vector_ranked, fts_ranked],
+        top_k=k,
+        k=settings.search_rrf_k,
+        key=lambda item: item,
+    )
+    return [
+        ("memory_node", payload_by_id[memory_id], float(score))
+        for memory_id, score in fused
+        if memory_id in payload_by_id
+    ]

memuron/search/pgvector.py

@@ -0,0 +1,252 @@
+"""PostgreSQL pgvector + HNSW helpers for memuron similarity search."""
+
+from __future__ import annotations
+
+import json
+import logging
+from typing import Any
+
+from artha_engine.store.projection_sql import sql_store_execute, sql_store_fetchall, sql_store_has_tables
+
+from memuron.application.config import settings
+from memuron.domain.scope_filter import scope_sql_clauses
+
+logger = logging.getLogger(__name__)
+
+_PGVECTOR_FLAG = "_memuron_pgvector_ready"
+_VECTOR_DIMS_FLAG = "_memuron_vector_dims"
+
+
+def vector_literal(values: list[float]) -> str:
+    return "[" + ",".join(f"{float(value):.8g}" for value in values) + "]"
+
+
+def pgvector_is_ready(store: object) -> bool:
+    return bool(getattr(store, _PGVECTOR_FLAG, False))
+
+
+def ensure_pgvector_schema(store: object, dimensions: int | None = None) -> bool:
+    """Enable pgvector, add embedding columns, backfill, and create HNSW indexes."""
+    if not sql_store_has_tables(store):
+        return False
+    if pgvector_is_ready(store):
+        return True
+
+    dims = dimensions or settings.vector_dimensions
+    try:
+        sql_store_execute(store, "CREATE EXTENSION IF NOT EXISTS vector")
+        sql_store_execute(
+            store,
+            f"ALTER TABLE memuron_memories ADD COLUMN IF NOT EXISTS embedding vector({dims})",
+        )
+        sql_store_execute(
+            store,
+            f"ALTER TABLE memuron_links ADD COLUMN IF NOT EXISTS embedding vector({dims})",
+        )
+        _backfill_table_embeddings(store, "memuron_memories", "artha_id", dims)
+        _backfill_table_embeddings(store, "memuron_links", "link_id", dims)
+        sql_store_execute(
+            store,
+            """
+            CREATE INDEX IF NOT EXISTS idx_memuron_memories_embedding_hnsw
+            ON memuron_memories USING hnsw (embedding vector_cosine_ops)
+            """,
+        )
+        sql_store_execute(
+            store,
+            """
+            CREATE INDEX IF NOT EXISTS idx_memuron_links_embedding_hnsw
+            ON memuron_links USING hnsw (embedding vector_cosine_ops)
+            """,
+        )
+        setattr(store, _PGVECTOR_FLAG, True)
+        setattr(store, _VECTOR_DIMS_FLAG, dims)
+        from memuron.search.fulltext import ensure_fulltext_schema
+
+        ensure_fulltext_schema(store)
+        logger.info("pgvector HNSW ready (dims=%s)", dims)
+        return True
+    except Exception as exc:
+        logger.warning("pgvector setup skipped: %s", exc)
+        return False
+
+
+def _backfill_table_embeddings(
+    store: object,
+    table: str,
+    id_column: str,
+    dimensions: int,
+) -> None:
+    rows = sql_store_fetchall(
+        store,
+        f"""
+        SELECT {id_column} AS row_id, embedding_json
+        FROM {table}
+        WHERE embedding IS NULL AND embedding_json IS NOT NULL AND embedding_json != '[]'
+        """,
+    )
+    for row in rows:
+        embedding = _parse_embedding(row.get("embedding_json"))
+        if not embedding or len(embedding) != dimensions:
+            continue
+        sql_store_execute(
+            store,
+            f"UPDATE {table} SET embedding = ?::vector WHERE {id_column} = ?",
+            (vector_literal(embedding), row["row_id"]),
+        )
+
+
+def _parse_embedding(value: Any) -> list[float]:
+    if isinstance(value, list):
+        return [float(item) for item in value]
+    if isinstance(value, str):
+        try:
+            parsed = json.loads(value)
+        except json.JSONDecodeError:
+            return []
+        if isinstance(parsed, list):
+            return [float(item) for item in parsed]
+    return []
+
+
+def _vector_dims(store: object) -> int:
+    return int(getattr(store, _VECTOR_DIMS_FLAG, settings.vector_dimensions))
+
+
+def sync_memory_embedding(store: object, artha_id: str, embedding: list[float]) -> None:
+    if not pgvector_is_ready(store) or not embedding:
+        return
+    dims = _vector_dims(store)
+    if len(embedding) != dims:
+        return
+    sql_store_execute(
+        store,
+        "UPDATE memuron_memories SET embedding = ?::vector WHERE artha_id = ?",
+        (vector_literal(embedding), artha_id),
+    )
+
+
+def sync_link_embedding(store: object, link_id: str, embedding: list[float]) -> None:
+    if not pgvector_is_ready(store) or not embedding:
+        return
+    dims = _vector_dims(store)
+    if len(embedding) != dims:
+        return
+    sql_store_execute(
+        store,
+        "UPDATE memuron_links SET embedding = ?::vector WHERE link_id = ?",
+        (vector_literal(embedding), link_id),
+    )
+
+
+def _scope_sql_clauses(column: str, scope: list[str] | None) -> tuple[str, list[str]]:
+    """Build ANDed scope glob filters applied before vector ranking (MemBrain-style)."""
+    return scope_sql_clauses(column, scope)
+
+
+def pgvector_memory_search(
+    store: object,
+    query_vector: list[float],
+    *,
+    top_k: int,
+    scope: list[str] | None = None,
+    include_content: bool = False,
+) -> list[dict[str, Any]]:
+    if not pgvector_is_ready(store) or not query_vector:
+        return []
+    if len(query_vector) != _vector_dims(store):
+        return []
+
+    query_literal = vector_literal(query_vector)
+    content_col = ", content, metadata_json" if include_content else ""
+    scope_sql, scope_params = _scope_sql_clauses("scope_json", scope)
+    scope_where = f" AND {scope_sql}" if scope_sql else ""
+    rows = sql_store_fetchall(
+        store,
+        f"""
+        SELECT artha_id, scope_json{content_col},
+               1 - (embedding <=> ?::vector) AS semantic_score
+        FROM memuron_memories
+        WHERE embedding IS NOT NULL{scope_where}
+        ORDER BY embedding <=> ?::vector
+        LIMIT ?
+        """,
+        (query_literal, *scope_params, query_literal, top_k),
+    )
+    hits: list[dict[str, Any]] = []
+    for row in rows:
+        hit: dict[str, Any] = {
+            "artha_id": str(row["artha_id"]),
+            "semantic_score": float(row["semantic_score"]),
+        }
+        if include_content:
+            hit["content"] = row.get("content")
+            hit["scope_json"] = row.get("scope_json")
+            hit["metadata_json"] = row.get("metadata_json")
+        hits.append(hit)
+    return hits
+
+
+def pgvector_memory_search_ids(
+    store: object,
+    query_vector: list[float],
+    *,
+    top_k: int,
+    scope: list[str] | None = None,
+) -> list[tuple[str, float]]:
+    rows = pgvector_memory_search(
+        store,
+        query_vector,
+        top_k=top_k,
+        scope=scope,
+        include_content=False,
+    )
+    return [(row["artha_id"], row["semantic_score"]) for row in rows]
+
+
+def pgvector_link_search(
+    store: object,
+    query_vector: list[float],
+    *,
+    top_k: int,
+    scope: list[str] | None = None,
+) -> list[dict[str, Any]]:
+    if not pgvector_is_ready(store) or not query_vector:
+        return []
+    if len(query_vector) != _vector_dims(store):
+        return []
+
+    query_literal = vector_literal(query_vector)
+    source_scope_sql, source_scope_params = _scope_sql_clauses("src.scope_json", scope)
+    target_scope_sql, target_scope_params = _scope_sql_clauses("tgt.scope_json", scope)
+    scope_where = ""
+    scope_params: list[str] = []
+    if source_scope_sql:
+        scope_where = f" AND {source_scope_sql} AND {target_scope_sql}"
+        scope_params = [*source_scope_params, *target_scope_params]
+
+    rows = sql_store_fetchall(
+        store,
+        f"""
+        SELECT l.link_id, l.source_id, l.target_id, l.description, l.metadata_json,
+               1 - (l.embedding <=> ?::vector) AS semantic_score
+        FROM memuron_links l
+        JOIN memuron_memories src ON src.artha_id = l.source_id
+        JOIN memuron_memories tgt ON tgt.artha_id = l.target_id
+        WHERE l.embedding IS NOT NULL{scope_where}
+        ORDER BY l.embedding <=> ?::vector
+        LIMIT ?
+        """,
+        (query_literal, *scope_params, query_literal, top_k),
+    )
+    return [
+        {
+            "link_id": str(row["link_id"]),
+            "source_id": str(row["source_id"]),
+            "target_id": str(row["target_id"]),
+            "description": str(row.get("description") or ""),
+            "metadata_json": row.get("metadata_json"),
+            "semantic_score": float(row["semantic_score"]),
+        }
+        for row in rows
+    ]

memuron/security/__init__.py

@@ -0,0 +1 @@
+"""Authentication, authorization, tenant, and Clerk integration."""