minder-cli 0.2.0__py3-none-any.whl

minder/retrieval/mmr.py

@@ -0,0 +1,116 @@
+"""
+Maximal Marginal Relevance (MMR) diversity re-ranking.
+
+MMR balances relevance to the query against redundancy among already-selected
+results.  lambda_mult controls the trade-off:
+  lambda_mult = 1.0  →  pure relevance ranking  (no diversity)
+  lambda_mult = 0.0  →  maximum diversity        (no relevance)
+  lambda_mult = 0.5  →  balanced default
+
+Reference: Carbonell & Goldstein (1998) "The Use of MMR, Diversity-Based
+           Reranking for Reordering Documents and Producing Summaries"
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Any
+
+
+# ---------------------------------------------------------------------------
+# Internal helpers
+# ---------------------------------------------------------------------------
+
+
+def _cosine(a: list[float], b: list[float]) -> float:
+    if not a or not b or len(a) != len(b):
+        return 0.0
+    dot = sum(x * y for x, y in zip(a, b, strict=False))
+    norm_a = math.sqrt(sum(x * x for x in a))
+    norm_b = math.sqrt(sum(x * x for x in b))
+    if norm_a == 0.0 or norm_b == 0.0:
+        return 0.0
+    return dot / (norm_a * norm_b)
+
+
+def _relevance(doc: dict[str, Any], query_embedding: list[float], embedding_key: str, score_key: str) -> float:
+    emb = doc.get(embedding_key)
+    if isinstance(emb, list) and emb:
+        return _cosine(query_embedding, emb)
+    return float(doc.get(score_key, 0.0))
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+
+def mmr_rerank(
+    query_embedding: list[float],
+    candidates: list[dict[str, Any]],
+    *,
+    top_k: int = 5,
+    lambda_mult: float = 0.5,
+    embedding_key: str = "embedding",
+    score_key: str = "score",
+) -> list[dict[str, Any]]:
+    """
+    Re-rank *candidates* using Maximal Marginal Relevance.
+
+    Args:
+        query_embedding: embedding vector of the query.
+        candidates: list of document dicts, each optionally containing an
+            ``embedding_key`` field (list[float]) and/or a ``score_key``
+            field (float).  Documents without embeddings fall back to their
+            existing score for relevance estimation.
+        top_k: maximum number of results to return.
+        lambda_mult: trade-off coefficient in [0, 1].
+        embedding_key: key in each doc dict that holds the document embedding.
+        score_key: key used as a relevance proxy when no embedding is present.
+
+    Returns:
+        A sub-list of *candidates* of length ≤ top_k, ordered by MMR score.
+    """
+    if not candidates:
+        return []
+
+    top_k = max(1, top_k)
+    lambda_mult = max(0.0, min(1.0, lambda_mult))
+
+    # Pre-compute relevance scores to avoid repeated cosine calls.
+    relevances = [
+        _relevance(doc, query_embedding, embedding_key, score_key)
+        for doc in candidates
+    ]
+
+    selected_indices: list[int] = []
+    remaining_indices = list(range(len(candidates)))
+
+    while remaining_indices and len(selected_indices) < top_k:
+        best_score = -float("inf")
+        best_pos = 0
+
+        for pos, idx in enumerate(remaining_indices):
+            rel = relevances[idx]
+
+            if not selected_indices:
+                mmr_score = rel
+            else:
+                doc_emb = candidates[idx].get(embedding_key)
+                if isinstance(doc_emb, list) and doc_emb:
+                    max_sim = max(
+                        _cosine(doc_emb, candidates[sel].get(embedding_key) or [])
+                        for sel in selected_indices
+                    )
+                else:
+                    max_sim = 0.0
+                mmr_score = lambda_mult * rel - (1.0 - lambda_mult) * max_sim
+
+            if mmr_score > best_score:
+                best_score = mmr_score
+                best_pos = pos
+
+        chosen = remaining_indices.pop(best_pos)
+        selected_indices.append(chosen)
+
+    return [candidates[i] for i in selected_indices]

minder/retrieval/multi_hop.py

@@ -0,0 +1,115 @@
+"""
+Multi-hop retrieval.
+
+Iteratively refines the search query using content from the previous hop's
+top result, then merges and de-duplicates results across all hops.
+
+Hop 1 → retrieve on original query
+Hop 2 → expand query with key terms extracted from hop-1 top result → retrieve
+...
+Final → merge all hops, sort by score, return top-K.
+"""
+
+from __future__ import annotations
+
+from collections import Counter
+from typing import Any, Protocol, runtime_checkable
+
+
+# ---------------------------------------------------------------------------
+# Retriever protocol
+# ---------------------------------------------------------------------------
+
+
+@runtime_checkable
+class RetrieveFn(Protocol):
+    """Any async callable ``(query, *, limit) → list[dict]`` qualifies."""
+
+    async def __call__(
+        self,
+        query: str,
+        *,
+        limit: int,
+    ) -> list[dict[str, Any]]: ...
+
+
+# ---------------------------------------------------------------------------
+# MultiHopRetriever
+# ---------------------------------------------------------------------------
+
+
+class MultiHopRetriever:
+    """
+    Iterative retrieval that uses the top result from each hop to expand the
+    query for the next hop.
+
+    Args:
+        retrieve_fn: async callable matching :class:`RetrieveFn`.
+        max_hops: total number of retrieval hops (default 2).
+    """
+
+    def __init__(self, retrieve_fn: RetrieveFn, *, max_hops: int = 2) -> None:
+        self._retrieve_fn = retrieve_fn
+        self._max_hops = max(1, max_hops)
+
+    async def retrieve(
+        self,
+        query: str,
+        *,
+        limit: int = 5,
+    ) -> list[dict[str, Any]]:
+        """
+        Run multi-hop retrieval.
+
+        Returns:
+            Deduplicated, score-sorted list of documents across all hops,
+            truncated to *limit*.  Each document gains a ``"hop"`` metadata
+            field indicating which hop first found it.
+        """
+        seen_keys: set[str] = set()
+        all_results: list[dict[str, Any]] = []
+        current_query = query
+
+        for hop in range(self._max_hops):
+            hop_results = await self._retrieve_fn(current_query, limit=limit)
+            new_this_hop: list[dict[str, Any]] = []
+            for doc in hop_results:
+                key = self._doc_key(doc)
+                if key not in seen_keys:
+                    seen_keys.add(key)
+                    enriched = dict(doc)
+                    enriched.setdefault("hop", hop + 1)
+                    all_results.append(enriched)
+                    new_this_hop.append(enriched)
+
+            # Expand query for next hop using key terms from top new result
+            if hop < self._max_hops - 1 and new_this_hop:
+                top_content = str(new_this_hop[0].get("content", ""))
+                expansion = self._expand_query(top_content, base_query=query)
+                if expansion:
+                    current_query = f"{query} {expansion}"
+
+        # Sort by descending score then stable insertion order
+        all_results.sort(key=lambda d: float(d.get("score", 0.0)), reverse=True)
+        return all_results[:limit]
+
+    # ------------------------------------------------------------------
+    # Internals
+    # ------------------------------------------------------------------
+
+    @staticmethod
+    def _doc_key(doc: dict[str, Any]) -> str:
+        return str(doc.get("path", doc.get("id", id(doc))))
+
+    @staticmethod
+    def _expand_query(content: str, *, base_query: str, max_terms: int = 5) -> str:
+        """Extract high-frequency content terms not already in the base query."""
+        base_tokens = set(base_query.lower().split())
+        tokens = [
+            tok
+            for tok in content.lower().split()
+            if len(tok) > 3 and tok not in base_tokens and tok.isalpha()
+        ]
+        freq: Counter[str] = Counter(tokens)
+        top_terms = [term for term, _ in freq.most_common(max_terms)]
+        return " ".join(top_terms)

minder/runtime.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+import importlib.util
+from typing import Any
+
+
+def module_available(module_name: str) -> bool:
+    return importlib.util.find_spec(module_name) is not None
+
+
+def load_attr(module_name: str, attr_name: str) -> Any | None:
+    if not module_available(module_name):
+        return None
+    module = __import__(module_name, fromlist=[attr_name])
+    return getattr(module, attr_name, None)

minder/server.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import asyncio
+import sys
+from pathlib import Path
+
+from minder.bootstrap.providers import (
+    build_cache,
+    build_graph_store,
+    build_store,
+    build_vector_store,
+)
+from minder.bootstrap.transport import build_transport
+from minder.config import Settings
+from minder.embedding.local import LocalEmbeddingProvider
+from minder.graph.runtime import graph_runtime_name
+from minder.llm.local import LocalModelLLM
+from minder.llm.openai import OpenAIFallbackLLM
+from minder.presentation.http.admin.routes import build_http_app, build_http_routes
+
+__all__ = [
+    "build_cache",
+    "build_graph_store",
+    "build_http_app",
+    "build_http_routes",
+    "build_store",
+    "build_transport",
+    "build_vector_store",
+    "main",
+    "runtime_summary",
+]
+
+
+def runtime_summary(config: Settings) -> dict[str, object]:
+    llm = LocalModelLLM(
+        config.llm.model_path,
+        runtime="auto",
+        context_length=config.llm.context_length,
+    )
+    embedder = LocalEmbeddingProvider(
+        config.embedding.model_path,
+        dimensions=config.embedding.dimensions,
+        runtime="auto",
+    )
+    fallback = OpenAIFallbackLLM(
+        config.llm.openai_api_key, config.llm.openai_model, runtime="auto"
+    )
+    return {
+        "transport": config.server.transport,
+        "host": config.server.host,
+        "port": config.server.port,
+        "orchestration_runtime_requested": config.workflow.orchestration_runtime,
+        "orchestration_runtime_effective": graph_runtime_name(
+            config.workflow.orchestration_runtime
+        ),
+        "llm_model_path": str(Path(config.llm.model_path).expanduser()),
+        "llm_runtime_effective": llm.runtime,
+        "llm_context_length": config.llm.context_length,
+        "embedding_model_path": str(Path(config.embedding.model_path).expanduser()),
+        "embedding_runtime_effective": embedder.runtime,
+        "openai_fallback_configured": fallback.available(),
+        "openai_fallback_runtime_effective": fallback.runtime,
+    }
+
+
+async def _async_run() -> None:
+    print("MINDER SERVER STARTING", file=sys.stderr, flush=True)
+    config = Settings()
+
+    # Initialise structured JSON logging and tracing before anything else
+    from minder.observability import configure_json_logging, configure_tracing
+
+    configure_json_logging(level=config.server.log_level)
+    configure_tracing(
+        service_name=config.server.name,
+        service_version=config.server.version,
+    )
+
+    store = build_store(config)
+    await store.init_db()
+
+    graph_store = build_graph_store(config)
+    if graph_store is not None and hasattr(graph_store, "init_db"):
+        await graph_store.init_db()
+
+    vector_store = build_vector_store(config, store)
+    if hasattr(vector_store, "setup"):
+        await vector_store.setup()
+
+    cache = build_cache(config)
+    admin = await store.get_user_by_username("admin")
+    print(f"MINDER ADMIN EXISTS: {admin is not None}", file=sys.stderr, flush=True)
+
+    transport = build_transport(
+        config=config,
+        store=store,
+        vector_store=vector_store,
+        graph_store=graph_store,
+        cache=cache,
+    )
+
+    from minder.prompts import PromptRegistry
+
+    await PromptRegistry.sync(transport.app, store)
+
+    print(
+        f"Minder store={config.relational_store.provider} cache={config.cache.provider} "
+        f"transport={transport.transport_name} host={config.server.host}:{config.server.port}",
+        file=sys.stderr,
+        flush=True,
+    )
+    print(
+        "Minder runtime summary:", runtime_summary(config), file=sys.stderr, flush=True
+    )
+
+    try:
+        if transport.transport_name == "stdio":
+            await transport.app.run_stdio_async()
+        else:
+            print(
+                f"Starting SSE on {config.server.host}:{config.server.port}",
+                file=sys.stderr,
+                flush=True,
+            )
+            if hasattr(transport, "run"):
+                await transport.run()
+            else:
+                await transport.app.run_sse_async()
+    finally:
+        await store.dispose()
+        if graph_store is not None and hasattr(graph_store, "dispose"):
+            await graph_store.dispose()
+        await cache.close()
+
+
+def _run() -> None:
+    asyncio.run(_async_run())
+
+
+def main() -> None:
+    _run()
+
+
+if __name__ == "__main__":
+    main()

minder/store/__init__.py

@@ -0,0 +1,64 @@
+"""
+Store package — data access layer.
+
+Exports both concrete implementations and domain interfaces.
+Application code should depend on interfaces from `minder.store.interfaces`.
+"""
+
+from .document import DocumentStore
+from .error import ErrorStore
+from .feedback import FeedbackStore
+from .graph import KnowledgeGraphStore
+from .history import HistoryStore
+from .interfaces import (
+    ICacheProvider,
+    IClientRepository,
+    IDocumentRepository,
+    IErrorRepository,
+    IFeedbackRepository,
+    IGraphRepository,
+    IHistoryRepository,
+    IOperationalStore,
+    IRepositoryRepo,
+    IRuleRepository,
+    ISessionRepository,
+    ISkillRepository,
+    IUserRepository,
+    IVectorStore,
+    IWorkflowRepository,
+    IWorkflowStateRepository,
+)
+from .relational import RelationalStore
+from .repo_state import RepoStateStore
+from .rule import RuleStore
+from .vector import VectorStore
+
+__all__ = [
+    # Domain interfaces
+    "ICacheProvider",
+    "IClientRepository",
+    "IDocumentRepository",
+    "IErrorRepository",
+    "IFeedbackRepository",
+    "IGraphRepository",
+    "IHistoryRepository",
+    "IOperationalStore",
+    "IRepositoryRepo",
+    "IRuleRepository",
+    "ISessionRepository",
+    "ISkillRepository",
+    "IUserRepository",
+    "IVectorStore",
+    "IWorkflowRepository",
+    "IWorkflowStateRepository",
+    # Concrete implementations
+    "DocumentStore",
+    "ErrorStore",
+    "FeedbackStore",
+    "HistoryStore",
+    "KnowledgeGraphStore",
+    "RelationalStore",
+    "RepoStateStore",
+    "RuleStore",
+    "VectorStore",
+]

minder/store/document.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+import uuid
+from typing import Any
+
+from sqlalchemy import delete, select, update
+
+from minder.models.document import Document
+from minder.store.relational import RelationalStore
+
+
+class DocumentStore:
+    def __init__(self, store: RelationalStore) -> None:
+        self._store = store
+
+    async def create_document(
+        self,
+        title: str,
+        content: str,
+        doc_type: str,
+        source_path: str,
+        project: str,
+        *,
+        chunks: dict[str, Any] | None = None,
+        embedding: list[float] | None = None,
+    ) -> Document:
+        async with self._store._session() as sess:
+            document = Document(
+                id=uuid.uuid4(),
+                title=title,
+                content=content,
+                doc_type=doc_type,
+                source_path=source_path,
+                chunks=chunks or {},
+                embedding=embedding,
+                project=project,
+            )
+            sess.add(document)
+            await sess.flush()
+            await sess.refresh(document)
+            return document
+
+    async def get_document_by_path(
+        self, source_path: str, *, project: str | None = None
+    ) -> Document | None:
+        async with self._store._session() as sess:
+            stmt = select(Document).where(Document.source_path == source_path)
+            if project is not None:
+                stmt = stmt.where(Document.project == project)
+            result = await sess.execute(stmt)
+            return result.scalar_one_or_none()
+
+    async def get_documents_by_ids(self, doc_ids: list[uuid.UUID]) -> list[Document]:
+        if not doc_ids:
+            return []
+        async with self._store._session() as sess:
+            stmt = select(Document).where(Document.id.in_(doc_ids))
+            result = await sess.execute(stmt)
+            return list(result.scalars().all())
+
+    async def list_documents(self, project: str | None = None) -> list[Document]:
+        async with self._store._session() as sess:
+            stmt = select(Document)
+            if project is not None:
+                stmt = stmt.where(Document.project == project)
+            result = await sess.execute(stmt)
+            return list(result.scalars().all())
+
+    async def upsert_document(
+        self,
+        *,
+        title: str,
+        content: str,
+        doc_type: str,
+        source_path: str,
+        project: str,
+        chunks: dict[str, Any] | None = None,
+        embedding: list[float] | None = None,
+    ) -> Document:
+        existing = await self.get_document_by_path(source_path, project=project)
+        if existing is None:
+            return await self.create_document(
+                title=title,
+                content=content,
+                doc_type=doc_type,
+                source_path=source_path,
+                project=project,
+                chunks=chunks,
+                embedding=embedding,
+            )
+
+        async with self._store._session() as sess:
+            await sess.execute(
+                update(Document)
+                .where(Document.id == existing.id)
+                .values(
+                    title=title,
+                    content=content,
+                    doc_type=doc_type,
+                    chunks=chunks or {},
+                    embedding=embedding,
+                    project=project,
+                )
+            )
+            result = await sess.execute(select(Document).where(Document.id == existing.id))
+            return result.scalar_one()
+
+    async def delete_documents_not_in_paths(
+        self, *, project: str, keep_paths: set[str]
+    ) -> None:
+        async with self._store._session() as sess:
+            stmt = delete(Document).where(Document.project == project)
+            if keep_paths:
+                stmt = stmt.where(Document.source_path.not_in(keep_paths))
+            await sess.execute(stmt)

minder/store/error.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+import math
+import uuid
+from collections import Counter
+from typing import Any
+from typing import cast
+
+from sqlalchemy import select
+
+from minder.models.error import Error
+from minder.store.relational import RelationalStore
+
+
+class ErrorStore:
+    def __init__(self, store: RelationalStore) -> None:
+        self._store = store
+
+    async def create_error(
+        self,
+        error_code: str,
+        error_message: str,
+        stack_trace: str | None = None,
+        context: dict[str, Any] | None = None,
+        resolution: str | None = None,
+        embedding: list[float] | None = None,
+        resolved: bool = False,
+    ) -> Error:
+        async with self._store._session() as sess:
+            error = Error(
+                id=uuid.uuid4(),
+                error_code=error_code,
+                error_message=error_message,
+                stack_trace=stack_trace,
+                context=context or {},
+                resolution=resolution,
+                embedding=embedding,
+                resolved=resolved,
+            )
+            sess.add(error)
+            await sess.flush()
+            await sess.refresh(error)
+            return error
+
+    async def list_errors(self) -> list[Error]:
+        async with self._store._session() as sess:
+            result = await sess.execute(select(Error))
+            return list(result.scalars().all())
+
+    async def search_errors(self, query: str, limit: int = 5) -> list[dict[str, Any]]:
+        rows = await self.list_errors()
+        query_vector = self._text_vector(query)
+        ranked = []
+        for row in rows:
+            text = f"{row.error_code} {row.error_message} {row.context}"
+            score = self._cosine_similarity(query_vector, self._text_vector(text))
+            ranked.append(
+                {
+                    "id": row.id,
+                    "error_code": row.error_code,
+                    "error_message": row.error_message,
+                    "resolution": row.resolution,
+                    "score": round(score, 4),
+                }
+            )
+        ranked.sort(key=lambda item: cast(float, item["score"]), reverse=True)
+        return ranked[:limit]
+
+    @staticmethod
+    def _text_vector(text: str) -> Counter[str]:
+        return Counter(token for token in text.lower().split() if len(token) > 2)
+
+    @staticmethod
+    def _cosine_similarity(left: Counter[str], right: Counter[str]) -> float:
+        if not left or not right:
+            return 0.0
+        numerator = sum(left[key] * right[key] for key in left.keys() & right.keys())
+        left_norm = math.sqrt(sum(value * value for value in left.values()))
+        right_norm = math.sqrt(sum(value * value for value in right.values()))
+        if left_norm == 0 or right_norm == 0:
+            return 0.0
+        return numerator / (left_norm * right_norm)