code-context-control 2.28.0__py3-none-any.whl

services/vector_store.py

@@ -0,0 +1,299 @@
+"""SLTM Vector Store with local fallback indexing."""
+
+from __future__ import annotations
+
+import json
+import threading
+import uuid
+from datetime import datetime, timezone
+from pathlib import Path
+
+from services.ollama_client import OllamaClient
+from services.text_index import TextIndex
+
+SLTM_CATEGORIES = [
+    "design_docs",
+    "api_contracts",
+    "bug_history",
+    "terminal_summaries",
+    "code_notes",
+    "general",
+]
+
+
+class VectorStore:
+    """Hybrid TF-IDF + vector search over categorized memory collections."""
+
+    def __init__(self, project_path: str, config: dict | None = None):
+        self.project_path = Path(project_path)
+        self.config = config or {}
+        self._lock = threading.Lock()
+
+        base_url = self.config.get("ollama_base_url", "http://localhost:11434")
+        self.ollama = OllamaClient(base_url)
+        self.embed_model = self.config.get("embed_model", "nomic-embed-text")
+        self.alpha = self.config.get("sltm_alpha", 0.5)
+
+        self._chroma_client = None
+        self._collections: dict[str, object] = {}
+        self._chroma_available = False
+        self._ollama_available = False
+        self._init_backends()
+
+        self._fallback_dir = self.project_path / ".c3" / "sltm" / "fallback"
+        self._fallback_dir.mkdir(parents=True, exist_ok=True)
+        self._fallback_data: dict[str, list[dict]] = {}
+        self._records_by_id: dict[str, dict] = {}
+        self._text_index = TextIndex()
+        self._load_fallback()
+
+    def _init_backends(self):
+        if self.config.get("disable_vector_backend"):
+            self._chroma_available = False
+            self._ollama_available = False
+            return
+        try:
+            import chromadb
+            from chromadb.config import Settings
+
+            persist_dir = str(self.project_path / ".c3" / "sltm" / "chromadb")
+            Path(persist_dir).mkdir(parents=True, exist_ok=True)
+            self._chroma_client = chromadb.PersistentClient(
+                path=persist_dir,
+                settings=Settings(anonymized_telemetry=False),
+            )
+            for cat in SLTM_CATEGORIES:
+                self._collections[cat] = self._chroma_client.get_or_create_collection(
+                    name=cat,
+                    metadata={"hnsw:space": "cosine"},
+                )
+            self._chroma_available = True
+        except Exception:
+            self._chroma_available = False
+
+        self._ollama_available = self.ollama.is_available(timeout=2) and self.ollama.has_model(self.embed_model)
+
+    @property
+    def vector_enabled(self) -> bool:
+        return self._chroma_available and self._ollama_available
+
+    def add(
+        self,
+        text: str,
+        category: str = "general",
+        metadata: dict | None = None,
+        record_id: str | None = None,
+    ) -> dict:
+        """Store a record in the SLTM using a stable record id when provided."""
+        if category not in SLTM_CATEGORIES:
+            category = "general"
+
+        record_id = record_id or uuid.uuid4().hex[:12]
+        now = datetime.now(timezone.utc).isoformat()
+        meta = dict(metadata or {})
+        meta.update({"timestamp": now, "category": category})
+        record = {
+            "id": record_id,
+            "text": text,
+            "metadata": meta,
+            "timestamp": now,
+            "category": category,
+        }
+
+        with self._lock:
+            self._delete_locked(record_id)
+            self._fallback_data.setdefault(category, []).append(record)
+            self._save_fallback_category(category)
+            self._records_by_id[record_id] = record
+            self._text_index.add_or_update(record_id, self._searchable_text(record))
+
+            if self.vector_enabled:
+                try:
+                    embedding = self.ollama.embed(text, model=self.embed_model)
+                    if embedding:
+                        flat_meta = {
+                            key: value if isinstance(value, (int, float, bool)) else str(value)
+                            for key, value in meta.items()
+                        }
+                        self._collections[category].add(
+                            ids=[record_id],
+                            embeddings=[embedding],
+                            documents=[text],
+                            metadatas=[flat_meta],
+                        )
+                except Exception:
+                    pass
+
+        return {
+            "stored": True,
+            "id": record_id,
+            "category": category,
+            "vector_indexed": self.vector_enabled,
+            "total_records": len(self._records_by_id),
+        }
+
+    def search(self, query: str, category: str = "", top_k: int = 5) -> list[dict]:
+        categories = [category] if category and category in SLTM_CATEGORIES else list(SLTM_CATEGORIES)
+        allowed = set(categories)
+
+        docs = {
+            doc_id: record
+            for doc_id, record in self._records_by_id.items()
+            if record.get("category") in allowed
+        }
+        if not docs:
+            return []
+
+        tfidf_scores = {
+            doc_id: score
+            for doc_id, score in self._text_index.search(query, top_k=max(top_k * 5, 20))
+            if doc_id in docs
+        }
+
+        vector_scores: dict[str, float] = {}
+        if self.vector_enabled:
+            try:
+                query_embedding = self.ollama.embed(query, model=self.embed_model)
+                if query_embedding:
+                    for cat in categories:
+                        col = self._collections.get(cat)
+                        if not col or col.count() == 0:
+                            continue
+                        results = col.query(
+                            query_embeddings=[query_embedding],
+                            n_results=min(max(top_k * 3, top_k), col.count()),
+                        )
+                        ids = (results or {}).get("ids") or []
+                        distances = (results or {}).get("distances") or []
+                        for i, rid in enumerate(ids[0] if ids else []):
+                            dist = distances[0][i] if distances and distances[0] else 0
+                            vector_scores[rid] = max(vector_scores.get(rid, 0.0), max(0.0, 1.0 - dist))
+            except Exception:
+                pass
+
+        candidate_ids = set(tfidf_scores) | set(vector_scores)
+        if not candidate_ids:
+            return []
+
+        max_tfidf = max(tfidf_scores.values()) if tfidf_scores else 1.0
+        max_tfidf = max(max_tfidf, 0.001)
+        min_score = float(self.config.get("sltm_min_score", 0.3))
+        combined = []
+        for doc_id in candidate_ids:
+            if doc_id not in docs:
+                continue
+            tfidf_score = tfidf_scores.get(doc_id, 0.0) / max_tfidf
+            vector_score = vector_scores.get(doc_id, 0.0)
+            score = self.alpha * tfidf_score + (1 - self.alpha) * vector_score if vector_scores else tfidf_score
+            if score >= min_score:
+                combined.append((doc_id, score))
+        combined.sort(key=lambda item: item[1], reverse=True)
+
+        results = []
+        for doc_id, score in combined[:top_k]:
+            record = docs[doc_id]
+            results.append({
+                "id": doc_id,
+                "text": record.get("text", ""),
+                "category": record.get("category", "general"),
+                "score": round(score, 4),
+                "metadata": record.get("metadata", {}),
+                "timestamp": record.get("timestamp", ""),
+                "search_method": "hybrid" if vector_scores else "tfidf",
+            })
+        return results
+
+    def delete(self, record_id: str) -> dict:
+        deleted = False
+        with self._lock:
+            deleted = self._delete_locked(record_id)
+        return {"deleted": deleted, "id": record_id}
+
+    def get_stats(self) -> dict:
+        stats = {
+            "vector_enabled": self.vector_enabled,
+            "chromadb_available": self._chroma_available,
+            "ollama_available": self._ollama_available,
+            "embed_model": self.embed_model,
+            "alpha": self.alpha,
+            "collections": {},
+            "total_records": len(self._records_by_id),
+        }
+        for cat in SLTM_CATEGORIES:
+            fallback_count = len(self._fallback_data.get(cat, []))
+            chroma_count = 0
+            if self._chroma_available and cat in self._collections:
+                try:
+                    chroma_count = self._collections[cat].count()
+                except Exception:
+                    pass
+            stats["collections"][cat] = {
+                "fallback_count": fallback_count,
+                "chroma_count": chroma_count,
+            }
+        return stats
+
+    def _delete_locked(self, record_id: str) -> bool:
+        deleted = False
+        existing = self._records_by_id.pop(record_id, None)
+        if existing:
+            self._text_index.remove(record_id)
+        for cat in SLTM_CATEGORIES:
+            records = self._fallback_data.get(cat, [])
+            kept = [record for record in records if record.get("id") != record_id]
+            if len(kept) != len(records):
+                self._fallback_data[cat] = kept
+                self._save_fallback_category(cat)
+                deleted = True
+            if self._chroma_available and cat in self._collections:
+                try:
+                    self._collections[cat].delete(ids=[record_id])
+                except Exception:
+                    pass
+        return deleted or existing is not None
+
+    def _searchable_text(self, record: dict) -> str:
+        metadata = record.get("metadata", {})
+        fields = [
+            record.get("text", ""),
+            record.get("category", ""),
+            metadata.get("source", ""),
+            metadata.get("fact_id", ""),
+            metadata.get("source_session", ""),
+        ]
+        return " ".join(str(field) for field in fields if field)
+
+    def _load_fallback(self):
+        docs = {}
+        self._records_by_id = {}
+        for cat in SLTM_CATEGORIES:
+            path = self._fallback_dir / f"{cat}.json"
+            if path.exists():
+                try:
+                    with open(path, encoding="utf-8") as handle:
+                        records = json.load(handle)
+                except Exception:
+                    records = []
+            else:
+                records = []
+            normalized = []
+            for record in records:
+                record = {
+                    "id": record.get("id"),
+                    "text": record.get("text", ""),
+                    "metadata": dict(record.get("metadata", {})),
+                    "timestamp": record.get("timestamp", ""),
+                    "category": record.get("category", cat),
+                }
+                if not record["id"]:
+                    continue
+                normalized.append(record)
+                self._records_by_id[record["id"]] = record
+                docs[record["id"]] = self._searchable_text(record)
+            self._fallback_data[cat] = normalized
+        self._text_index.rebuild(docs)
+
+    def _save_fallback_category(self, category: str):
+        path = self._fallback_dir / f"{category}.json"
+        with open(path, "w", encoding="utf-8") as handle:
+            json.dump(self._fallback_data.get(category, []), handle, indent=2)

services/version_tracker.py

@@ -0,0 +1,271 @@
+"""VersionTracker - Git-aware version tracking for key project files."""
+import hashlib
+import json
+import subprocess
+import sys
+from datetime import datetime, timezone
+from pathlib import Path
+
+from core.ide import get_profile
+
+
+class VersionTracker:
+    """Track key file versions with Git metadata when available."""
+
+    DEFAULT_MAX_KEY_FILES = 10
+    HISTORY_LIMIT = 12
+
+    def __init__(self, project_path: str, ide_name: str = "claude-code"):
+        self.project_path = Path(project_path).resolve()
+        self.ide_name = ide_name
+        self.store_path = self.project_path / ".c3" / "version_tracker.json"
+        self.store_path.parent.mkdir(parents=True, exist_ok=True)
+        self._git_root = self._detect_git_root()
+        self.state = self._load_state()
+
+    def scan(self, agent: str = "current", max_files: int | None = None) -> dict:
+        target = self.ide_name if agent in ("", "current", None) else str(agent)
+        key_files = self.discover_key_files(agent=target, max_files=max_files)
+        tracked = {}
+        changed = []
+        now = datetime.now(timezone.utc).isoformat()
+
+        for item in key_files:
+            rel_path = item["file"]
+            record = self._build_record(rel_path, item.get("reason", ""))
+            tracked[rel_path] = record
+            prev = (self.state.get("files") or {}).get(rel_path)
+            if prev and self._record_signature(prev) != self._record_signature(record):
+                change = {
+                    "file": rel_path,
+                    "reason": record.get("reason", ""),
+                    "exists": record.get("exists", False),
+                    "git": record.get("git", {}),
+                    "previous_hash": prev.get("sha256", ""),
+                    "current_hash": record.get("sha256", ""),
+                }
+                record["history"] = ([change] + prev.get("history", []))[:self.HISTORY_LIMIT]
+                changed.append(change)
+            elif prev:
+                record["history"] = prev.get("history", [])[:self.HISTORY_LIMIT]
+            else:
+                record["history"] = []
+
+        self.state = {
+            "updated_at": now,
+            "ide_name": self.ide_name,
+            "agent": target,
+            "git_root": str(self._git_root) if self._git_root else "",
+            "files": tracked,
+        }
+        self._save_state()
+        return {
+            "agent": target,
+            "updated_at": now,
+            "git_available": bool(self._git_root),
+            "files": list(tracked.values()),
+            "changed": changed,
+        }
+
+    def get_status(self, agent: str = "current", changed_only: bool = False, max_files: int | None = None) -> dict:
+        result = self.scan(agent=agent, max_files=max_files)
+        if changed_only:
+            changed_paths = {item["file"] for item in result["changed"]}
+            result["files"] = [item for item in result["files"] if item["file"] in changed_paths]
+        return result
+
+    def discover_key_files(self, agent: str = "current", max_files: int | None = None) -> list[dict]:
+        profile = get_profile(agent if agent not in ("", "current", None) else self.ide_name)
+        seen = set()
+        files: list[dict] = []
+
+        def add(rel_path: str, reason: str) -> None:
+            rel = str(Path(rel_path)).replace("\\", "/")
+            if rel.startswith("./"):
+                rel = rel[2:]
+            if not rel or rel in seen:
+                return
+            files.append({"file": rel, "reason": reason})
+            seen.add(rel)
+
+        add(".c3/config.json", "C3 project configuration")
+        if profile.instructions_file:
+            add(profile.instructions_file, f"{profile.display_name} instructions")
+        if profile.config_path and not profile.config_path_global:
+            add(profile.config_path, f"{profile.display_name} MCP config")
+        if profile.settings_path:
+            add(profile.settings_path, f"{profile.display_name} settings")
+
+        for rel_path, reason in self._hot_files():
+            add(rel_path, reason)
+
+        conventional = [
+            ("README.md", "Project overview"),
+            ("cli/c3.py", "CLI entry point"),
+            ("cli/mcp_server.py", "MCP server entry"),
+            ("services/agents.py", "Background agent logic"),
+            ("core/config.py", "Shared defaults"),
+        ]
+        for rel_path, reason in conventional:
+            if (self.project_path / rel_path).exists():
+                add(rel_path, reason)
+
+        limit = max(1, int(max_files or self.DEFAULT_MAX_KEY_FILES))
+        return files[:limit]
+
+    def _hot_files(self) -> list[tuple[str, str]]:
+        session_dir = self.project_path / ".c3" / "sessions"
+        if not session_dir.exists():
+            return []
+        counts: dict[str, int] = {}
+        for sf in sorted(session_dir.glob("session_*.json"), reverse=True)[:20]:
+            try:
+                with open(sf, encoding="utf-8") as f:
+                    session = json.load(f)
+                for ft in session.get("files_touched", []):
+                    rel_path = str(ft.get("file", "") or "").replace("\\", "/")
+                    if rel_path:
+                        counts[rel_path] = counts.get(rel_path, 0) + 1
+            except Exception:
+                continue
+        ranked = sorted(counts.items(), key=lambda item: (-item[1], item[0]))
+        return [(path, f"edited in {count} sessions") for path, count in ranked if count >= 2][:5]
+
+    def _build_record(self, rel_path: str, reason: str) -> dict:
+        path = self.project_path / rel_path
+        exists = path.exists()
+        size = path.stat().st_size if exists else 0
+        mtime = path.stat().st_mtime if exists else 0
+        return {
+            "file": rel_path,
+            "reason": reason,
+            "exists": exists,
+            "size": size,
+            "mtime": mtime,
+            "sha256": self._sha256(path) if exists and path.is_file() else "",
+            "git": self._git_info(rel_path),
+        }
+
+    def _record_signature(self, record: dict) -> str:
+        git = record.get("git", {}) or {}
+        return "|".join([
+            str(int(bool(record.get("exists")))),
+            str(record.get("size", 0)),
+            str(record.get("mtime", 0)),
+            record.get("sha256", ""),
+            git.get("commit", ""),
+            str(int(bool(git.get("dirty")))),
+        ])
+
+    def _sha256(self, path: Path) -> str:
+        try:
+            return hashlib.sha256(path.read_bytes()).hexdigest()[:16]
+        except Exception:
+            return ""
+
+    def _detect_git_root(self) -> Path | None:
+        try:
+            result = subprocess.run(
+                ["git", "rev-parse", "--show-toplevel"],
+                cwd=self.project_path,
+                capture_output=True,
+                text=True,
+                timeout=3,
+                check=True,
+            )
+            root = (result.stdout or "").strip()
+            return Path(root).resolve() if root else None
+        except Exception:
+            return None
+
+    def _git_info(self, rel_path: str) -> dict:
+        info = {
+            "available": bool(self._git_root),
+            "tracked": False,
+            "dirty": False,
+            "commit": "",
+            "author": "",
+            "timestamp": 0,
+            "subject": "",
+        }
+        if not self._git_root:
+            return info
+        abs_path = (self.project_path / rel_path).resolve()
+        try:
+            git_rel = abs_path.relative_to(self._git_root)
+        except Exception:
+            return info
+        try:
+            kwargs = {}
+            if sys.platform == "win32":
+                kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+            subprocess.run(
+                ["git", "ls-files", "--error-unmatch", str(git_rel)],
+                cwd=self._git_root,
+                capture_output=True,
+                text=True,
+                timeout=3,
+                check=True,
+                **kwargs
+            )
+            info["tracked"] = True
+        except Exception:
+            return info
+        try:
+            kwargs = {}
+            if sys.platform == "win32":
+                kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+            status = subprocess.run(
+                ["git", "status", "--porcelain", "--", str(git_rel)],
+                cwd=self._git_root,
+                capture_output=True,
+                text=True,
+                timeout=3,
+                check=True,
+                **kwargs
+            )
+            info["dirty"] = bool((status.stdout or "").strip())
+        except Exception:
+            pass
+        try:
+            kwargs = {}
+            if sys.platform == "win32":
+                kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW
+            log = subprocess.run(
+                ["git", "log", "-1", "--format=%H%x1f%ct%x1f%an%x1f%s", "--", str(git_rel)],
+                cwd=self._git_root,
+                capture_output=True,
+                text=True,
+                timeout=3,
+                check=True,
+                **kwargs
+            )
+            parts = (log.stdout or "").strip().split("\x1f")
+            if len(parts) == 4:
+                info["commit"] = parts[0]
+                try:
+                    info["timestamp"] = int(parts[1])
+                except Exception:
+                    info["timestamp"] = 0
+                info["author"] = parts[2]
+                info["subject"] = parts[3]
+        except Exception:
+            pass
+        return info
+
+    def _load_state(self) -> dict:
+        if not self.store_path.exists():
+            return {"files": {}}
+        try:
+            with open(self.store_path, encoding="utf-8") as f:
+                data = json.load(f)
+            if isinstance(data, dict):
+                data.setdefault("files", {})
+                return data
+        except Exception:
+            pass
+        return {"files": {}}
+
+    def _save_state(self) -> None:
+        with open(self.store_path, "w", encoding="utf-8") as f:
+            json.dump(self.state, f, indent=2)