minder-cli 0.2.0__py3-none-any.whl

minder/store/milvus/vector_store.py

@@ -0,0 +1,232 @@
+"""
+Milvus Vector Store — implements IVectorStore using PyMilvus.
+
+All operations execute in a thread pool since PyMilvus is synchronous.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+import uuid
+from typing import Any
+
+from minder.store.interfaces import IDocumentRepository
+from minder.store.milvus.client import MilvusClient
+from minder.store.milvus.collections import get_document_schema
+
+
+logger = logging.getLogger(__name__)
+
+
+class MilvusVectorStore:
+    def __init__(
+        self,
+        client: MilvusClient,
+        document_store: IDocumentRepository,
+        prefix: str = "minder_",
+        dimensions: int = 768,
+    ) -> None:
+        self._client = client
+        self._document_store = document_store
+        self._prefix = prefix
+        self._dimensions = dimensions
+        self._doc_collection = f"{prefix}documents"
+        self._collection_ready = False
+        self._collection_lock = asyncio.Lock()
+
+    async def setup(self) -> None:
+        await self._ensure_collection_ready(force=True)
+
+    async def _ensure_collection_ready(self, *, force: bool = False) -> None:
+        if self._collection_ready and not force:
+            return
+
+        async with self._collection_lock:
+            if self._collection_ready and not force:
+                return
+
+            loop = asyncio.get_running_loop()
+
+            def _setup() -> None:
+                if self._client.client.has_collection(self._doc_collection):
+                    metadata = self._client.client.describe_collection(self._doc_collection)
+                    current_dim = self._extract_embedding_dimension(metadata)
+                    if current_dim is not None and current_dim != self._dimensions:
+                        logger.warning(
+                            "Milvus collection %s dimension mismatch: existing=%s expected=%s; recreating collection for reindex",
+                            self._doc_collection,
+                            current_dim,
+                            self._dimensions,
+                        )
+                        self._client.client.drop_collection(self._doc_collection)
+
+                if not self._client.client.has_collection(self._doc_collection):
+                    self._create_collection()
+
+            await loop.run_in_executor(None, _setup)
+            self._collection_ready = True
+
+    def _create_collection(self) -> None:
+        schema = get_document_schema(self._dimensions)
+        index_params = self._client.client.prepare_index_params()
+        index_params.add_index(
+            field_name="embedding",
+            index_type="AUTOINDEX",
+            metric_type="COSINE",
+        )
+        self._client.client.create_collection(
+            collection_name=self._doc_collection,
+            schema=schema,
+            index_params=index_params,
+        )
+
+    @staticmethod
+    def _extract_embedding_dimension(metadata: Any) -> int | None:
+        if not isinstance(metadata, dict):
+            return None
+        fields = metadata.get("fields", [])
+        if not isinstance(fields, list):
+            return None
+
+        for field in fields:
+            if not isinstance(field, dict):
+                continue
+            field_name = field.get("name") or field.get("field_name")
+            if field_name != "embedding":
+                continue
+
+            for key in ("params", "type_params", "element_type_params"):
+                params = field.get(key)
+                if isinstance(params, dict) and params.get("dim") is not None:
+                    return int(params["dim"])
+
+            if field.get("dim") is not None:
+                return int(field["dim"])
+
+        return None
+
+    def _validate_embedding_length(self, embedding: list[float]) -> None:
+        if len(embedding) != self._dimensions:
+            raise ValueError(
+                f"Embedding length {len(embedding)} does not match configured Milvus dimension {self._dimensions}"
+            )
+
+    @staticmethod
+    def _serialize_payload(payload: dict[str, Any]) -> dict[str, Any]:
+        return {
+            "title": str(payload.get("title", "")),
+            "source_path": str(payload.get("source_path", payload.get("path", ""))),
+            "doc_type": str(payload.get("doc_type", "")),
+            "project": str(payload.get("project", "")),
+        }
+
+    async def upsert_document(
+        self,
+        doc_id: uuid.UUID,
+        embedding: list[float],
+        payload: dict[str, Any],
+    ) -> None:
+        self._validate_embedding_length(embedding)
+        await self._ensure_collection_ready()
+        loop = asyncio.get_running_loop()
+
+        def _upsert() -> None:
+            self._client.client.upsert(
+                collection_name=self._doc_collection,
+                data=[
+                    {
+                        "id": str(doc_id),
+                        "embedding": embedding,
+                        "project": payload.get("project", ""),
+                        "doc_type": payload.get("doc_type", ""),
+                        "payload": self._serialize_payload(payload),
+                    }
+                ],
+            )
+
+        await loop.run_in_executor(None, _upsert)
+
+    async def delete_documents(self, doc_ids: list[uuid.UUID]) -> None:
+        if not doc_ids:
+            return
+        loop = asyncio.get_running_loop()
+
+        def _delete() -> None:
+            id_list = [f"'{did}'" for did in doc_ids]
+            expr = f"id in [{', '.join(id_list)}]"
+            self._client.client.delete(
+                collection_name=self._doc_collection,
+                filter=expr,
+            )
+
+        await loop.run_in_executor(None, _delete)
+
+    async def search_documents(
+        self,
+        query_embedding: list[float],
+        *,
+        project: str | None = None,
+        doc_types: set[str] | None = None,
+        limit: int = 5,
+        score_threshold: float = 0.0,
+    ) -> list[dict[str, Any]]:
+        self._validate_embedding_length(query_embedding)
+        await self._ensure_collection_ready()
+        loop = asyncio.get_running_loop()
+
+        filter_expr = ""
+        filters = []
+        if project:
+            filters.append(f'project == "{project}"')
+        if doc_types:
+            types_str = ", ".join(f'"{t}"' for t in doc_types)
+            filters.append(f"doc_type in [{types_str}]")
+
+        if filters:
+            filter_expr = " and ".join(filters)
+
+        def _search() -> Any:
+            return self._client.client.search(
+                collection_name=self._doc_collection,
+                data=[query_embedding],
+                filter=filter_expr,
+                limit=limit,
+                output_fields=["payload"],
+            )
+
+        results = await loop.run_in_executor(None, _search)
+
+        hits_with_payload: list[tuple[Any, dict[str, Any]]] = []
+        doc_ids: list[uuid.UUID] = []
+        ranked: list[dict[str, Any]] = []
+        if results and len(results) > 0:
+            for hit in results[0]:
+                if hit.distance < score_threshold:
+                    continue
+                payload = hit.entity.get("payload", {})
+                doc_id = uuid.UUID(hit.id) if isinstance(hit.id, str) else hit.id
+                if isinstance(doc_id, uuid.UUID):
+                    doc_ids.append(doc_id)
+                hits_with_payload.append((hit, payload if isinstance(payload, dict) else {}))
+
+        docs_by_id = {
+            doc.id: doc for doc in await self._document_store.get_documents_by_ids(doc_ids)
+        }
+
+        for hit, payload in hits_with_payload:
+            doc_id = uuid.UUID(hit.id) if isinstance(hit.id, str) else hit.id
+            doc = docs_by_id.get(doc_id) if isinstance(doc_id, uuid.UUID) else None
+            ranked.append(
+                {
+                    "id": doc_id,
+                    "title": getattr(doc, "title", None) or payload.get("title", ""),
+                    "path": getattr(doc, "source_path", None)
+                    or payload.get("source_path", payload.get("path", "")),
+                    "content": getattr(doc, "content", None) or "",
+                    "score": round(hit.distance, 4),
+                    "doc_type": getattr(doc, "doc_type", None) or payload.get("doc_type", ""),
+                }
+            )
+
+        return ranked

minder/store/mongodb/__init__.py

@@ -0,0 +1,11 @@
+"""
+MongoDB Store Package — Motor-based async adapters implementing domain interfaces.
+"""
+
+from minder.store.mongodb.client import MongoClient
+from minder.store.mongodb.operational_store import MongoOperationalStore
+
+__all__ = [
+    "MongoClient",
+    "MongoOperationalStore",
+]

minder/store/mongodb/client.py

@@ -0,0 +1,49 @@
+"""
+MongoDB Async Client — Motor singleton with connection pooling.
+
+Usage:
+    client = MongoClient(uri="mongodb://localhost:27017", database="minder")
+    db = client.db
+    await client.health_check()
+    await client.close()
+"""
+
+from __future__ import annotations
+
+from motor.motor_asyncio import AsyncIOMotorClient, AsyncIOMotorDatabase
+
+
+class MongoClient:
+    """Thin wrapper around Motor's async client with lifecycle helpers."""
+
+    def __init__(
+        self,
+        uri: str = "mongodb://localhost:27017",
+        database: str = "minder",
+        *,
+        min_pool_size: int = 5,
+        max_pool_size: int = 50,
+    ) -> None:
+        self._client: AsyncIOMotorClient = AsyncIOMotorClient(  # type: ignore[type-arg]
+            uri,
+            minPoolSize=min_pool_size,
+            maxPoolSize=max_pool_size,
+            serverSelectionTimeoutMS=5000,
+        )
+        self._db: AsyncIOMotorDatabase = self._client[database]  # type: ignore[type-arg]
+
+    @property
+    def db(self) -> AsyncIOMotorDatabase:  # type: ignore[type-arg]
+        return self._db
+
+    async def health_check(self) -> bool:
+        """Ping the MongoDB server."""
+        try:
+            await self._client.admin.command("ping")
+            return True
+        except Exception:
+            return False
+
+    async def close(self) -> None:
+        """Close the Motor client."""
+        self._client.close()

minder/store/mongodb/indexes.py

@@ -0,0 +1,90 @@
+"""
+MongoDB collection indexes — called once at application startup.
+"""
+
+from __future__ import annotations
+
+from motor.motor_asyncio import AsyncIOMotorDatabase
+from pymongo import ASCENDING
+
+
+async def ensure_indexes(db: AsyncIOMotorDatabase) -> None:  # type: ignore[type-arg]
+    """Create all required MongoDB indexes (idempotent)."""
+
+    # Users
+    users = db["users"]
+    await users.create_index([("email", ASCENDING)], unique=True)
+    await users.create_index([("username", ASCENDING)], unique=True)
+    await users.create_index([("company_id", ASCENDING)])
+
+    # Clients
+    clients = db["clients"]
+    await clients.create_index([("slug", ASCENDING)], unique=True)
+    await clients.create_index([("created_by_user_id", ASCENDING)])
+
+    client_api_keys = db["client_api_keys"]
+    await client_api_keys.create_index([("client_id", ASCENDING)])
+    await client_api_keys.create_index([("status", ASCENDING)])
+
+    client_sessions = db["client_sessions"]
+    await client_sessions.create_index([("client_id", ASCENDING)])
+    await client_sessions.create_index([("access_token_id", ASCENDING)], unique=True)
+
+    audit_logs = db["audit_logs"]
+    await audit_logs.create_index([("actor_id", ASCENDING)])
+    await audit_logs.create_index([("event_type", ASCENDING)])
+
+    admin_jobs = db["admin_jobs"]
+    await admin_jobs.create_index([("job_type", ASCENDING)])
+    await admin_jobs.create_index([("status", ASCENDING)])
+    await admin_jobs.create_index([("requested_by_user_id", ASCENDING)])
+    await admin_jobs.create_index([("created_at", ASCENDING)])
+
+    # Skills
+    skills = db["skills"]
+    await skills.create_index([("company_id", ASCENDING)])
+    await skills.create_index([("title", ASCENDING)])
+    await skills.create_index([("language", ASCENDING)])
+
+    # Prompts
+    prompts = db["prompts"]
+    await prompts.create_index([("company_id", ASCENDING)])
+    await prompts.create_index([("name", ASCENDING)], unique=True)
+
+    # Sessions
+    sessions = db["sessions"]
+    await sessions.create_index([("company_id", ASCENDING)])
+    await sessions.create_index([("user_id", ASCENDING)])
+    await sessions.create_index([("repo_id", ASCENDING)])
+
+    # Workflows
+    workflows = db["workflows"]
+    await workflows.create_index([("company_id", ASCENDING)])
+    await workflows.create_index([("name", ASCENDING)])
+
+    # Repositories
+    repos = db["repositories"]
+    await repos.create_index([("company_id", ASCENDING)])
+    await repos.create_index([("repo_name", ASCENDING)])
+
+    # Repository Workflow States
+    workflow_states = db["repository_workflow_states"]
+    await workflow_states.create_index([("repo_id", ASCENDING)])
+    await workflow_states.create_index([("session_id", ASCENDING)])
+
+    # Documents
+    documents = db["documents"]
+    await documents.create_index([("doc_type", ASCENDING)])
+    await documents.create_index([("project", ASCENDING)])
+    await documents.create_index(
+        [("source_path", ASCENDING), ("project", ASCENDING)],
+        unique=True,
+    )
+
+    # History
+    history = db["history"]
+    await history.create_index([("session_id", ASCENDING)])
+
+    # Errors
+    errors = db["errors"]
+    await errors.create_index([("error_code", ASCENDING)])