minder-cli 0.2.0__py3-none-any.whl

minder/tools/memory.py

@@ -0,0 +1,381 @@
+from __future__ import annotations
+
+import math
+import uuid
+from typing import Any, TYPE_CHECKING
+
+from minder.continuity import compatibility_score_for_memory
+from minder.config import MinderConfig
+from minder.embedding.local import LocalEmbeddingProvider
+from minder.observability.metrics import record_continuity_recall
+from minder.store.interfaces import IOperationalStore
+
+if TYPE_CHECKING:
+    from minder.continuity import ContinuitySynthesizer
+
+
+class MemoryTools:
+    def __init__(self, store: IOperationalStore, config: MinderConfig) -> None:
+        self._store = store
+        self._config = config
+        self._embedder = LocalEmbeddingProvider(
+            config.embedding.model_path,
+            dimensions=min(config.embedding.dimensions, 16),
+            runtime="auto",
+        )
+        self._synthesizer: ContinuitySynthesizer | None = None
+
+    def _get_synthesizer(self) -> "ContinuitySynthesizer":
+        if self._synthesizer is None:
+            from minder.continuity import ContinuitySynthesizer
+
+            self._synthesizer = ContinuitySynthesizer(self._config)
+        return self._synthesizer
+
+    async def minder_memory_store(
+        self,
+        *,
+        title: str,
+        content: str,
+        tags: list[str],
+        language: str,
+    ) -> dict[str, Any]:
+        skill = await self._store.create_skill(
+            id=uuid.uuid4(),
+            title=title,
+            content=content,
+            language=language,
+            tags=tags,
+            embedding=self._embedder.embed(f"{title}\n{content}"),
+            usage_count=0,
+            quality_score=0.0,
+        )
+
+        # Record persistent audit event
+        try:
+            await self._store.create_audit_log(
+                actor_type="system",
+                actor_id="minder",
+                event_type="skill.created",
+                resource_type="skill",
+                resource_id=str(skill.id),
+                outcome="success",
+                audit_metadata={"title": title},
+            )
+        except Exception:
+            pass
+
+        return {"id": str(skill.id), "title": skill.title, "tags": list(skill.tags)}
+
+    async def minder_memory_recall(
+        self,
+        query: str,
+        *,
+        limit: int = 5,
+        current_step: str | None = None,
+        artifact_type: str | None = None,
+    ) -> list[dict[str, Any]]:
+        query_embedding = self._embedder.embed(query)
+        skills = await self._store.list_skills()
+        ranked: list[dict[str, Any]] = []
+        for skill in skills:
+            embedding = skill.embedding if isinstance(skill.embedding, list) else None
+            if not embedding:
+                continue
+            semantic_score = self._cosine_similarity(query_embedding, embedding)
+            compatibility_score, compatibility_reasons = compatibility_score_for_memory(
+                tags=list(skill.tags) if isinstance(skill.tags, list) else [],
+                title=str(skill.title),
+                content=str(skill.content),
+                current_step=current_step,
+                artifact_type=artifact_type,
+            )
+            score = min((semantic_score * 0.8) + (compatibility_score * 0.2), 1.0)
+            ranked.append(
+                {
+                    "id": str(skill.id),
+                    "title": skill.title,
+                    "content": skill.content,
+                    "tags": list(skill.tags) if isinstance(skill.tags, list) else [],
+                    "semantic_score": round(semantic_score, 4),
+                    "step_compatibility": round(compatibility_score, 4),
+                    "continuity_reasons": compatibility_reasons,
+                    "score": round(score, 4),
+                }
+            )
+        ranked.sort(key=lambda item: float(item["score"]), reverse=True)
+        limited = ranked[:limit]
+        synthesis, synthesis_meta = self._get_synthesizer().synthesize_memory_hits(
+            query=query,
+            hits=limited,
+            current_step=current_step,
+            artifact_type=artifact_type,
+        )
+        for item in limited:
+            item["recall_summary"] = synthesis["summary"]
+            item["hit_summary"] = synthesis["hit_summaries"].get(str(item["id"]), "")
+            item["synthesis"] = synthesis_meta
+            record_continuity_recall(
+                provider=str(synthesis_meta.get("provider", "unknown")),
+                step_compatibility=float(item["step_compatibility"]),
+            )
+        return limited
+
+    async def minder_memory_list(self) -> list[dict[str, Any]]:
+        skills = await self._store.list_skills()
+        return [
+            {
+                "id": str(skill.id),
+                "title": skill.title,
+                "language": skill.language,
+                "tags": list(skill.tags) if isinstance(skill.tags, list) else [],
+            }
+            for skill in skills
+        ]
+
+    async def minder_memory_delete(self, skill_id: str) -> dict[str, bool]:
+        await self._store.delete_skill(uuid.UUID(skill_id))
+
+        # Record persistent audit event
+        try:
+            await self._store.create_audit_log(
+                actor_type="system",
+                actor_id="minder",
+                event_type="skill.deleted",
+                resource_type="skill",
+                resource_id=skill_id,
+                outcome="success",
+            )
+        except Exception:
+            pass
+
+        return {"deleted": True}
+
+    async def minder_memory_compact(
+        self,
+        *,
+        memory_ids: list[str],
+        similarity_threshold: float = 0.92,
+        dry_run: bool = True,
+    ) -> dict[str, Any]:
+        normalized_ids = self._normalize_memory_ids(memory_ids)
+        if len(normalized_ids) < 2:
+            raise ValueError("At least two memory_ids are required for compaction")
+
+        records = []
+        for memory_id in normalized_ids:
+            skill = await self._store.get_skill_by_id(uuid.UUID(memory_id))
+            if skill is None:
+                raise ValueError(f"Memory not found: {memory_id}")
+            embedding = self._embedder.embed(
+                self._compaction_text(
+                    title=str(skill.title),
+                    content=str(skill.content),
+                )
+            )
+            records.append(
+                {
+                    "id": str(skill.id),
+                    "title": str(skill.title),
+                    "content": str(skill.content),
+                    "language": str(getattr(skill, "language", "") or "markdown"),
+                    "tags": list(getattr(skill, "tags", []) or []),
+                    "embedding": embedding,
+                    "usage_count": int(getattr(skill, "usage_count", 0) or 0),
+                    "quality_score": float(getattr(skill, "quality_score", 0.0) or 0.0),
+                    "created_at": getattr(skill, "created_at", None),
+                    "updated_at": getattr(skill, "updated_at", None),
+                }
+            )
+
+        groups = self._duplicate_groups(records, similarity_threshold)
+        plans = [
+            self._build_compaction_plan(group) for group in groups if len(group) > 1
+        ]
+        result: dict[str, Any] = {
+            "dry_run": dry_run,
+            "candidate_count": len(records),
+            "duplicate_group_count": len(plans),
+            "plans": plans,
+        }
+        if dry_run or not plans:
+            result["compacted_count"] = 0
+            result["deleted_count"] = 0
+            return result
+
+        compacted: list[dict[str, Any]] = []
+        deleted_count = 0
+        for plan in plans:
+            primary_id = str(plan["primary_id"])
+            primary = next(item for item in plan["members"] if item["id"] == primary_id)
+            merged_tags = sorted(
+                {
+                    str(tag).strip().lower()
+                    for member in plan["members"]
+                    for tag in list(member.get("tags", []) or [])
+                    if str(tag).strip()
+                }
+            )
+            merged_content = max(
+                [str(member.get("content", "") or "") for member in plan["members"]],
+                key=len,
+            )
+            merged_quality = max(
+                float(member.get("quality_score", 0.0) or 0.0)
+                for member in plan["members"]
+            )
+            merged_usage = sum(
+                int(member.get("usage_count", 0) or 0) for member in plan["members"]
+            )
+            updated = await self._store.update_skill(
+                uuid.UUID(primary_id),
+                content=merged_content,
+                tags=merged_tags,
+                usage_count=merged_usage,
+                quality_score=merged_quality,
+                embedding=self._embedder.embed(f"{primary['title']}\n{merged_content}"),
+            )
+            if updated is None:
+                raise ValueError(f"Memory not found during compaction: {primary_id}")
+
+            duplicate_ids = [
+                str(member["id"])
+                for member in plan["members"]
+                if str(member["id"]) != primary_id
+            ]
+            for duplicate_id in duplicate_ids:
+                await self._store.delete_skill(uuid.UUID(duplicate_id))
+                deleted_count += 1
+
+            try:
+                await self._store.create_audit_log(
+                    actor_type="system",
+                    actor_id="minder",
+                    event_type="skill.compacted",
+                    resource_type="skill",
+                    resource_id=primary_id,
+                    outcome="success",
+                    audit_metadata={
+                        "merged_ids": duplicate_ids,
+                        "similarity_threshold": similarity_threshold,
+                    },
+                )
+            except Exception:
+                pass
+
+            compacted.append(
+                {
+                    "primary_id": primary_id,
+                    "merged_ids": duplicate_ids,
+                    "merged_tags": merged_tags,
+                    "usage_count": merged_usage,
+                    "quality_score": round(merged_quality, 4),
+                }
+            )
+
+        result["compacted_count"] = len(compacted)
+        result["deleted_count"] = deleted_count
+        result["compacted"] = compacted
+        return result
+
+    @staticmethod
+    def _compaction_text(*, title: str, content: str) -> str:
+        normalized_content = str(content or "").strip()
+        if normalized_content:
+            return normalized_content
+        return str(title or "").strip()
+
+    @staticmethod
+    def _normalize_memory_ids(memory_ids: list[str]) -> list[str]:
+        normalized: list[str] = []
+        seen: set[str] = set()
+        for raw_id in memory_ids:
+            value = str(raw_id or "").strip()
+            if not value or value in seen:
+                continue
+            uuid.UUID(value)
+            seen.add(value)
+            normalized.append(value)
+        return normalized
+
+    @staticmethod
+    def _duplicate_groups(
+        records: list[dict[str, Any]], similarity_threshold: float
+    ) -> list[list[dict[str, Any]]]:
+        adjacency: dict[str, set[str]] = {
+            str(record["id"]): set() for record in records
+        }
+        record_map = {str(record["id"]): record for record in records}
+        for index, left in enumerate(records):
+            left_embedding = list(left.get("embedding") or [])
+            for right in records[index + 1 :]:
+                right_embedding = list(right.get("embedding") or [])
+                similarity = MemoryTools._cosine_similarity(
+                    left_embedding, right_embedding
+                )
+                if similarity < similarity_threshold:
+                    continue
+                left_id = str(left["id"])
+                right_id = str(right["id"])
+                adjacency[left_id].add(right_id)
+                adjacency[right_id].add(left_id)
+
+        groups: list[list[dict[str, Any]]] = []
+        visited: set[str] = set()
+        for record in records:
+            record_id = str(record["id"])
+            if record_id in visited:
+                continue
+            stack = [record_id]
+            component: list[dict[str, Any]] = []
+            while stack:
+                current = stack.pop()
+                if current in visited:
+                    continue
+                visited.add(current)
+                component.append(record_map[current])
+                stack.extend(sorted(adjacency[current] - visited))
+            groups.append(component)
+        return groups
+
+    def _build_compaction_plan(self, members: list[dict[str, Any]]) -> dict[str, Any]:
+        primary = max(members, key=self._primary_sort_key)
+        duplicate_ids = [
+            str(member["id"])
+            for member in members
+            if str(member["id"]) != str(primary["id"])
+        ]
+        return {
+            "primary_id": str(primary["id"]),
+            "primary_title": str(primary["title"]),
+            "duplicate_ids": duplicate_ids,
+            "duplicate_titles": [
+                str(member["title"])
+                for member in members
+                if str(member["id"]) != str(primary["id"])
+            ],
+            "members": members,
+        }
+
+    @staticmethod
+    def _primary_sort_key(member: dict[str, Any]) -> tuple[float, int, str, str, int]:
+        updated_at = member.get("updated_at")
+        created_at = member.get("created_at")
+        return (
+            float(member.get("quality_score", 0.0) or 0.0),
+            int(member.get("usage_count", 0) or 0),
+            str(updated_at or ""),
+            str(created_at or ""),
+            len(str(member.get("content", "") or "")),
+        )
+
+    @staticmethod
+    def _cosine_similarity(left: list[float], right: list[float]) -> float:
+        if not left or not right or len(left) != len(right):
+            return 0.0
+        numerator = sum(a * b for a, b in zip(left, right, strict=False))
+        left_norm = math.sqrt(sum(value * value for value in left))
+        right_norm = math.sqrt(sum(value * value for value in right))
+        if left_norm == 0 or right_norm == 0:
+            return 0.0
+        return numerator / (left_norm * right_norm)

minder/tools/query.py

@@ -0,0 +1,307 @@
+from __future__ import annotations
+
+from collections.abc import AsyncGenerator
+import uuid
+from pathlib import Path
+from typing import Any
+
+from minder.config import MinderConfig
+from minder.embedding.local import LocalEmbeddingProvider
+from minder.graph import GraphState, MinderGraph
+from minder.graph.nodes.retriever import RetrieverNode
+from minder.observability.metrics import (
+    record_continuity_packet,
+    record_query_prompt_render,
+)
+from minder.prompts import PromptRegistry
+from minder.store.interfaces import IOperationalStore, IVectorStore
+from minder.tools.graph import GraphTools
+from minder.tools.ingest import IngestTools
+
+
+class QueryTools:
+    def __init__(
+        self,
+        store: IOperationalStore,
+        config: MinderConfig,
+        graph: MinderGraph | None = None,
+        vector_store: IVectorStore | None = None,
+        graph_tools: GraphTools | None = None,
+    ) -> None:
+        from minder.store.vector import VectorStore
+
+        self._store = store
+        self._config = config
+        self._graph = graph or MinderGraph(store, config)
+        self._vector_store = vector_store or VectorStore(store, store)
+        self._embedding_provider = LocalEmbeddingProvider(
+            config.embedding.model_path,
+            dimensions=config.embedding.dimensions,
+            runtime="auto",
+        )
+        self._ingest_tools = IngestTools(
+            self._store,
+            self._embedding_provider,
+            vector_store=self._vector_store,
+        )
+        self._graph_tools = graph_tools
+
+    async def minder_query(
+        self,
+        query: str,
+        *,
+        repo_path: str | None,
+        session_id: uuid.UUID | None = None,
+        user_id: uuid.UUID | None = None,
+        repo_id: uuid.UUID | None = None,
+        workflow_name: str | None = None,
+        verification_payload: dict[str, Any] | None = None,
+        max_attempts: int = 2,
+        allowed_repo_scopes: list[str] | None = None,
+    ) -> dict[str, Any]:
+        project_name = Path(repo_path).name if repo_path else None
+        if repo_path:
+            await self._ingest_tools.minder_ingest_directory(
+                repo_path, project=project_name
+            )
+        workflow_context: dict[str, Any] = (
+            {"workflow_name": workflow_name} if workflow_name else {}
+        )
+        if self._graph_tools is not None and repo_path:
+            cross_repo_context, cross_repo_graph = (
+                await self._graph_tools.build_cross_repo_context(
+                    query,
+                    repo_path=repo_path,
+                    repo_id=str(repo_id) if repo_id is not None else None,
+                    repo_name=Path(repo_path).name,
+                    allowed_repo_scopes=allowed_repo_scopes,
+                )
+            )
+            if cross_repo_context:
+                workflow_context["cross_repo_context"] = cross_repo_context
+            if cross_repo_graph is not None:
+                workflow_context["cross_repo_graph"] = cross_repo_graph
+        query_prompt = await PromptRegistry.resolve_prompt_model(
+            "query_reasoning",
+            self._store,
+        )
+        state = GraphState(
+            query=query,
+            session_id=session_id,
+            user_id=user_id,
+            repo_id=repo_id,
+            repo_path=repo_path,
+            workflow_context=workflow_context,
+            metadata={
+                "verification_payload": verification_payload,
+                "max_attempts": max_attempts,
+                "project_name": project_name,
+                "query_prompt_name": getattr(query_prompt, "name", "query_reasoning"),
+                "query_prompt_template": getattr(query_prompt, "content_template", ""),
+                "query_prompt_defaults": dict(
+                    getattr(query_prompt, "defaults", {}) or {}
+                ),
+                "query_prompt_source": (
+                    "builtin"
+                    if bool(getattr(query_prompt, "is_builtin", False))
+                    else "custom"
+                ),
+            },
+        )
+        result = await self._graph.run(state)
+        record_continuity_packet("query")
+        record_query_prompt_render(
+            str(
+                result.metadata.get(
+                    "query_prompt_source",
+                    state.metadata.get("query_prompt_source", "unknown"),
+                )
+            ),
+            correction_retries=sum(
+                1
+                for item in result.transition_log
+                if str(item.get("edge")) == "guard_failed"
+            ),
+        )
+        return {
+            "answer": result.llm_output.get("text", ""),
+            "sources": result.reasoning_output.get("sources", []),
+            "workflow": result.workflow_context,
+            "guard_result": result.guard_result,
+            "verification_result": result.verification_result,
+            "evaluation": result.evaluation,
+            "provider": result.llm_output.get("provider"),
+            "model": result.llm_output.get(
+                "model", result.llm_output.get("model_path")
+            ),
+            "runtime": result.llm_output.get("runtime"),
+            "orchestration_runtime": result.metadata.get("orchestration_runtime"),
+            "transition_log": result.transition_log,
+            "edge": result.metadata.get("edge"),
+            "cross_repo_graph": result.workflow_context.get("cross_repo_graph"),
+        }
+
+    async def minder_query_stream(
+        self,
+        query: str,
+        *,
+        repo_path: str | None,
+        session_id: uuid.UUID | None = None,
+        user_id: uuid.UUID | None = None,
+        repo_id: uuid.UUID | None = None,
+        workflow_name: str | None = None,
+        verification_payload: dict[str, Any] | None = None,
+        max_attempts: int = 2,
+        allowed_repo_scopes: list[str] | None = None,
+    ) -> AsyncGenerator[dict[str, Any], None]:
+        project_name = Path(repo_path).name if repo_path else None
+        if repo_path:
+            await self._ingest_tools.minder_ingest_directory(
+                repo_path, project=project_name
+            )
+        workflow_context: dict[str, Any] = (
+            {"workflow_name": workflow_name} if workflow_name else {}
+        )
+        if self._graph_tools is not None and repo_path:
+            cross_repo_context, cross_repo_graph = (
+                await self._graph_tools.build_cross_repo_context(
+                    query,
+                    repo_path=repo_path,
+                    repo_id=str(repo_id) if repo_id is not None else None,
+                    repo_name=Path(repo_path).name,
+                    allowed_repo_scopes=allowed_repo_scopes,
+                )
+            )
+            if cross_repo_context:
+                workflow_context["cross_repo_context"] = cross_repo_context
+            if cross_repo_graph is not None:
+                workflow_context["cross_repo_graph"] = cross_repo_graph
+        query_prompt = await PromptRegistry.resolve_prompt_model(
+            "query_reasoning",
+            self._store,
+        )
+        state = GraphState(
+            query=query,
+            session_id=session_id,
+            user_id=user_id,
+            repo_id=repo_id,
+            repo_path=repo_path,
+            workflow_context=workflow_context,
+            metadata={
+                "verification_payload": verification_payload,
+                "max_attempts": max_attempts,
+                "project_name": project_name,
+                "query_prompt_name": getattr(query_prompt, "name", "query_reasoning"),
+                "query_prompt_template": getattr(query_prompt, "content_template", ""),
+                "query_prompt_defaults": dict(
+                    getattr(query_prompt, "defaults", {}) or {}
+                ),
+                "query_prompt_source": (
+                    "builtin"
+                    if bool(getattr(query_prompt, "is_builtin", False))
+                    else "custom"
+                ),
+            },
+        )
+
+        async for event in self._graph.stream(state):
+            if str(event.get("type")) == "final":
+                final_state = event.get("state")
+                if isinstance(final_state, GraphState):
+                    result = self._result_from_state(final_state)
+                    record_continuity_packet("query")
+                    record_query_prompt_render(
+                        str(
+                            final_state.metadata.get(
+                                "query_prompt_source",
+                                state.metadata.get("query_prompt_source", "unknown"),
+                            )
+                        ),
+                        correction_retries=sum(
+                            1
+                            for item in final_state.transition_log
+                            if str(item.get("edge")) == "guard_failed"
+                        ),
+                    )
+                    yield {"type": "final", "payload": result}
+                continue
+            yield event
+
+    def _result_from_state(self, result: GraphState) -> dict[str, Any]:
+        return {
+            "answer": result.llm_output.get("text", ""),
+            "sources": result.reasoning_output.get("sources", []),
+            "workflow": result.workflow_context,
+            "guard_result": result.guard_result,
+            "verification_result": result.verification_result,
+            "evaluation": result.evaluation,
+            "provider": result.llm_output.get("provider"),
+            "model": result.llm_output.get(
+                "model", result.llm_output.get("model_path")
+            ),
+            "runtime": result.llm_output.get("runtime"),
+            "orchestration_runtime": result.metadata.get("orchestration_runtime"),
+            "transition_log": result.transition_log,
+            "edge": result.metadata.get("edge"),
+            "cross_repo_graph": result.workflow_context.get("cross_repo_graph"),
+        }
+
+    async def minder_search_code(
+        self, query: str, *, repo_path: str, limit: int = 5
+    ) -> list[dict[str, Any]]:
+        await self._ingest_tools.minder_ingest_directory(
+            repo_path, project=Path(repo_path).name
+        )
+        project_name = Path(repo_path).name
+        semantic_code_hits = await self._vector_store.search_documents(
+            self._embedding_provider.embed(query),
+            project=project_name,
+            doc_types={"code"},
+            limit=limit,
+            score_threshold=0.0,
+        )
+        if semantic_code_hits:
+            return [
+                {
+                    "path": doc["path"],
+                    "title": doc["title"],
+                    "score": doc["score"],
+                    "source_type": doc.get("doc_type", "unknown"),
+                }
+                for doc in semantic_code_hits[:limit]
+            ]
+
+        state = GraphState(
+            query=query,
+            repo_path=repo_path,
+            metadata={"project_name": project_name},
+        )
+        retriever = RetrieverNode(
+            top_k=limit,
+            embedding_provider=self._embedding_provider,
+            vector_store=self._vector_store,
+            score_threshold=self._config.retrieval.similarity_threshold,
+        )
+        state = await retriever.run(state)
+        code_docs = [
+            doc for doc in state.retrieved_docs if doc.get("doc_type") == "code"
+        ]
+        docs_to_return = code_docs or state.retrieved_docs
+        return [
+            {
+                "path": doc["path"],
+                "title": doc["title"],
+                "score": doc["score"],
+                "source_type": doc.get("doc_type", "unknown"),
+            }
+            for doc in docs_to_return[:limit]
+        ]
+
+    async def minder_search_errors(
+        self, query: str, *, limit: int = 5
+    ) -> list[dict[str, Any]]:
+        return await self._store.search_errors(query, limit=limit)
+
+    @staticmethod
+    def discover_repo_files(repo_path: str) -> list[str]:
+        return [str(path) for path in Path(repo_path).rglob("*") if path.is_file()]