codegraph-cli 2.1.1__py3-none-any.whl → 2.1.3__py3-none-any.whl

codegraph_cli/crew_chat.py

@@ -4,6 +4,7 @@ from __future__ import annotations
 
 import logging
 import os
+import re
 from typing import TYPE_CHECKING, Dict, List
 
 from datetime import datetime
@@ -44,14 +45,24 @@ class CrewChatAgent:
         # ── Silence CrewAI / LiteLLM noise ────────────────
         for logger_name in (
             "crewai", "crewai.agents", "crewai.crews",
-            "crewai.tasks", "litellm", "httpx",
+            "crewai.tasks", "litellm", "litellm.proxy",
+            "litellm.litellm_core_utils", "litellm.utils",
+            "litellm.proxy.proxy_server", "httpx", "httpcore",
         ):
             logging.getLogger(logger_name).setLevel(logging.CRITICAL)
         os.environ.setdefault("CREWAI_TELEMETRY_OPT_OUT", "true")
+        # Suppress LiteLLM's fastapi proxy import spam
+        os.environ.setdefault("LITELLM_LOG", "ERROR")
+        try:
+            import litellm
+            litellm.suppress_debug_info = True
+            litellm.set_verbose = False
+        except ImportError:
+            pass
 
         # ── Provider env vars for LiteLLM compatibility ───
         if llm.api_key:
-            provider = (llm.provider or "").lower()
+            provider = (llm.provider_name or "").lower()
             if provider == "openrouter" or (llm.endpoint and "openrouter.ai" in llm.endpoint):
                 os.environ["OPENROUTER_API_KEY"] = llm.api_key
                 if llm.endpoint:
@@ -87,27 +98,53 @@ class CrewChatAgent:
         self.file_ops_agent = create_file_ops_agent(tools["file_ops"], crew_llm, project_ctx)
         self.code_gen_agent = create_code_gen_agent(tools["all"], crew_llm, project_ctx)
         self.code_analysis_agent = create_code_analysis_agent(tools["code_analysis"], crew_llm, project_ctx)
-        self.coordinator = create_coordinator_agent(crew_llm, project_ctx)
+        # Give coordinator ALL tools so it can read files, search code, and delegate
+        self.coordinator = create_coordinator_agent(crew_llm, project_ctx, tools=tools["all"])
+
+        # ── Conversation memory for multi-turn continuity ─
+        self._history: list[dict] = []
+        self._max_history_pairs: int = 10
 
     # ── Public API ────────────────────────────────────────
 
     def process_message(self, user_message: str) -> str:
-        """Process a user message via CrewAI multi-agent pipeline."""
+        """Process a user message via CrewAI multi-agent pipeline.
+
+        Injects conversation history into the task description so agents
+        can follow up on previous suggestions and requests.
+        """
         try:
             proj_summary = self.context.get_project_summary()
-            context_str = (
+
+            # ── Build context with project info + conversation history ──
+            context_parts = [
                 f"Project: {proj_summary.get('project_name', 'Unknown')}, "
                 f"Root: {proj_summary.get('source_path', '.')}, "
-                f"Indexed files: {proj_summary.get('indexed_files', 0)}\n\n"
-            )
+                f"Indexed files: {proj_summary.get('indexed_files', 0)}",
+            ]
+
+            history_block = self._format_history()
+            if history_block:
+                context_parts.append(
+                    "\n── PREVIOUS CONVERSATION (use this to understand follow-up requests) ──\n"
+                    f"{history_block}\n"
+                    "── END PREVIOUS CONVERSATION ──"
+                )
+
+            context_parts.append(f"\nCurrent User Request: {user_message}")
 
             task = Task(
-                description=f"{context_str}User Request: {user_message}",
+                description="\n".join(context_parts),
                 expected_output=(
-                    "A specific, concrete answer based on actual project files and code. "
-                    "Use tools to explore the project. For code changes, use write_file or "
-                    "patch_file tools which automatically create backups. "
-                    "Don't give generic explanations."
+                    "Concrete results based on actual project files.\n"
+                    "CRITICAL: If the user refers to something from the previous conversation "
+                    "(e.g. 'apply those changes', 'implement what you suggested', 'do it', "
+                    "'make those improvements'), you MUST look at the PREVIOUS CONVERSATION "
+                    "section above to understand exactly what was discussed, then ACT on it "
+                    "using write_file or patch_file tools.\n"
+                    "For code changes: MUST use write_file or patch_file tools to actually "
+                    "modify files. Don't just describe changes — actually apply them and "
+                    "confirm they were applied. After making changes, read the file to verify."
                 ),
                 agent=self.coordinator,
             )
@@ -125,11 +162,113 @@ class CrewChatAgent:
             )
 
             result = crew.kickoff()
-            return str(result.raw) if hasattr(result, "raw") else str(result)
+            raw = str(result.raw) if hasattr(result, "raw") else str(result)
+            response = self._clean_response(raw)
+
+            # ── Store in conversation memory ──
+            self._history.append({"role": "user", "content": user_message})
+            self._history.append({"role": "assistant", "content": response})
+            self._trim_history()
+
+            return response
 
         except Exception as e:
             return f"❌ Error: {e}\n\nPlease try rephrasing your request."
 
+    # ── Response sanitisation ─────────────────────────────
+
+    @staticmethod
+    def _clean_response(text: str) -> str:
+        """Strip LLM reasoning artifacts from the crew output.
+
+        Some models (e.g. DeepSeek, StepFun) put their real answer inside
+        <think>…</think> tags and only emit a tiny fragment afterwards.
+        If stripping <think> would leave less than 100 useful chars, we
+        extract the *content* of the think block as the real answer.
+        """
+        # ── Phase 1: Handle <think> blocks smartly ────────
+        think_match = re.search(r"<think>([\s\S]*?)</think>", text)
+        if think_match:
+            think_content = think_match.group(1).strip()
+            without_think = re.sub(r"<think>[\s\S]*?</think>", "", text)
+            without_think = re.sub(r"</?think>", "", without_think).strip()
+            # If the part outside <think> is too short, the real answer
+            # is inside the think block
+            if len(without_think) < 100 and len(think_content) > len(without_think):
+                text = think_content
+            else:
+                text = without_think
+        else:
+            # Stray opening / closing think tags (no matched pair)
+            text = re.sub(r"</?think>", "", text)
+
+        # ── Phase 2: Strip tool-call / XML artifacts ──────
+        text = re.sub(r"<tool_call>[\s\S]*?</tool_call>", "", text)
+        text = re.sub(r"</?tool_call>", "", text)
+        text = re.sub(
+            r"</?(?:function|parameter|result|output|observation)(?:=[^>]*)?>\s*",
+            "", text,
+        )
+
+        # ── Phase 3: Clean up whitespace ──────────────────
+        text = re.sub(r"\n{3,}", "\n\n", text)
+        return text.strip()
+
+    # ── Conversation memory helpers ───────────────────────
+
+    def _format_history(self) -> str:
+        """Format conversation history for injection into task context.
+
+        Keeps the last 4 exchanges (8 messages).  The most recent 2
+        exchanges are preserved in full (up to a cap); older ones are
+        compressed to ~400 chars each to stay within LLM context limits.
+        """
+        if not self._history:
+            return ""
+
+        # Show at most the last 8 messages (4 exchanges)
+        recent = self._history[-8:]
+        lines: list[str] = []
+        total = len(recent)
+
+        for i, msg in enumerate(recent):
+            role = "User" if msg["role"] == "user" else "Assistant"
+            content = msg["content"]
+            # Last 2 exchanges (4 msgs) get generous space; older ones compressed
+            is_recent = i >= total - 4
+            max_len = 1500 if is_recent else 400
+            if len(content) > max_len:
+                content = content[:max_len] + "\n... (truncated)"
+            lines.append(f"[{role}]:\n{content}")
+
+        if len(self._history) > 8:
+            lines.insert(0, f"... ({len(self._history) - 8} older messages omitted)")
+
+        return "\n\n".join(lines)
+
+    def _trim_history(self):
+        """Keep only the last N exchange pairs to avoid unbounded growth."""
+        max_messages = self._max_history_pairs * 2
+        if len(self._history) > max_messages:
+            self._history = self._history[-max_messages:]
+
+    def clear_history(self):
+        """Clear conversation memory (called on /clear and /new)."""
+        self._history.clear()
+
+    def load_session_history(self, session) -> None:
+        """Load conversation history from a persisted ChatSession.
+
+        Called when resuming a previous session so the crew retains
+        context from earlier exchanges.
+        """
+        self._history.clear()
+        for msg in session.messages:
+            role = msg.role if hasattr(msg, "role") else msg["role"]
+            content = msg.content if hasattr(msg, "content") else msg["content"]
+            self._history.append({"role": role, "content": content})
+        self._trim_history()
+
     # ── Rollback helpers (called from REPL) ───────────────
 
     def list_all_backups(self) -> List[Dict]:

codegraph_cli/crew_tools.py

@@ -18,7 +18,23 @@ except ImportError:
         def __init_subclass__(cls, **kwargs): pass
         def __init__(self, **kwargs): pass
     CREWAI_AVAILABLE = False
-from pydantic import BaseModel, Field, PrivateAttr
+
+try:
+    from pydantic import BaseModel, Field, PrivateAttr
+except ImportError:
+    from dataclasses import dataclass, field as _field
+
+    class BaseModel:  # type: ignore
+        def __init_subclass__(cls, **kwargs): pass
+        def __init__(self, **kwargs):
+            for k, v in kwargs.items():
+                setattr(self, k, v)
+
+    def Field(*args, **kwargs):  # type: ignore
+        return kwargs.get("default", None)
+
+    def PrivateAttr(*args, **kwargs):  # type: ignore
+        return kwargs.get("default", None)
 
 if TYPE_CHECKING:
     from .project_context import ProjectContext
@@ -192,7 +208,12 @@ class ReadFileTool(ContextTool):
             numbered = []
             for i, line in enumerate(content.split("\n"), 1):
                 numbered.append(f"{i:4d} │ {line}")
-            return f"── {path} ──\n" + "\n".join(numbered)
+            result = f"── {path} ──\n" + "\n".join(numbered)
+            # Truncate to avoid blowing up LLM context window
+            max_chars = 12_000
+            if len(result) > max_chars:
+                result = result[:max_chars] + f"\n... (truncated — file has {len(content)} chars total, use grep_in_project to find specific sections)"
+            return result
         except Exception as e:
             return f"Error reading file: {e}"
 
@@ -273,6 +294,10 @@ class DeleteFileTool(ContextTool):
             backup_id = _backup_file(ctx.source_path, path, tag="delete")
             full_path = ctx.source_path / path
             full_path.unlink()
+
+            # Remove stale nodes from the code graph
+            ctx.remove_from_index(path)
+
             return f"✅ Deleted {path}\n   Backup: {backup_id} (use rollback_file to restore)"
         except Exception as e:
             return f"Error deleting file: {e}"
@@ -334,16 +359,18 @@ class SearchCodeTool(BaseTool):
 
     def _run(self, query: str) -> str:
         try:
-            results = self._rag.search(query, top_k=8)
+            results = self._rag.search(query, top_k=5)
             if not results:
                 return f"No results found for: {query}"
             lines = [f"Found {len(results)} results for '{query}':\n"]
             for i, r in enumerate(results, 1):
+                # Truncate preview to keep context compact
+                preview = r.snippet[:200].replace("\n", " ")
                 lines.append(
                     f"{i}. [{r.node_type}] {r.qualname}\n"
                     f"   File: {r.file_path}:{r.start_line}-{r.end_line}\n"
                     f"   Score: {r.score:.2f}\n"
-                    f"   Preview: {r.snippet[:120]}…\n"
+                    f"   Preview: {preview}…\n"
                 )
             return "\n".join(lines)
         except Exception as e:

codegraph_cli/embeddings.py

@@ -1,6 +1,6 @@
 """Configurable code embedding engine with multiple model support.
 
-Supported models (configure via ``cg set-embedding``):
+Supported models (configure via ``cg config set-embedding``):
 
 ========== ====================================== ========= ====== ======================
 Key        HuggingFace Model                      Download  Dim    Notes
@@ -301,7 +301,7 @@ class HashEmbeddingModel:
 
     Provides basic keyword-level similarity. Used as the default when
     ``torch``/``transformers`` are not installed or when ``hash`` is
-    selected via ``cg set-embedding hash``.
+    selected via ``cg config set-embedding hash``.
     """
 
     def __init__(self, dim: int = 256) -> None:
@@ -396,14 +396,104 @@ def get_embedder(
 # ===================================================================
 
 def cosine_similarity(vec_a: List[float], vec_b: List[float]) -> float:
-    """Cosine similarity between two L2-normalised vectors."""
+    """Cosine similarity between two vectors.
+
+    If vectors are already L2-normalised the dot product *is* the
+    cosine similarity.  For safety this function still divides by
+    the product of norms so it works correctly even when vectors
+    are *not* pre-normalised.
+
+    Returns a value in ``[-1, 1]``.  Zero-length or mismatched
+    vectors return ``0.0``.
+    """
     if not vec_a or not vec_b or len(vec_a) != len(vec_b):
         return 0.0
-    return sum(a * b for a, b in zip(vec_a, vec_b))
+    dot = sum(a * b for a, b in zip(vec_a, vec_b))
+    norm_a = math.sqrt(sum(a * a for a in vec_a))
+    norm_b = math.sqrt(sum(b * b for b in vec_b))
+    if norm_a < 1e-12 or norm_b < 1e-12:
+        return 0.0
+    return dot / (norm_a * norm_b)
 
 
 def _l2_normalize(vec: List[float]) -> List[float]:
+    """L2-normalise *vec* in-place.  Returns zero vector unchanged."""
     norm = math.sqrt(sum(v * v for v in vec))
-    if norm == 0:
+    if norm < 1e-12:
         return vec
     return [v / norm for v in vec]
+
+
+def embedding_norm(vec: List[float]) -> float:
+    """Return the L2 norm of a vector."""
+    return math.sqrt(sum(v * v for v in vec))
+
+
+# ===================================================================
+# Debug / Validation Utilities
+# ===================================================================
+
+def validate_embedding(vec: List[float], label: str = "embedding") -> Dict[str, Any]:
+    """Validate an embedding vector and return diagnostic info.
+
+    Checks:
+    - Vector is non-empty
+    - Vector contains no NaN / Inf values
+    - L2 norm is approximately 1.0 (unit-normalised)
+    - Vector is not all-zeros
+
+    Returns a dict with ``ok``, ``norm``, ``dim``, and any ``warnings``.
+    """
+    info: Dict[str, Any] = {
+        "label": label,
+        "dim": len(vec),
+        "ok": True,
+        "warnings": [],
+    }
+
+    if not vec:
+        info["ok"] = False
+        info["warnings"].append("empty vector")
+        info["norm"] = 0.0
+        return info
+
+    norm = embedding_norm(vec)
+    info["norm"] = norm
+
+    if any(math.isnan(v) or math.isinf(v) for v in vec):
+        info["ok"] = False
+        info["warnings"].append("contains NaN or Inf")
+
+    if norm < 1e-9:
+        info["ok"] = False
+        info["warnings"].append("zero vector")
+    elif abs(norm - 1.0) > 0.01:
+        info["warnings"].append(f"not unit-normalised (norm={norm:.4f})")
+
+    return info
+
+
+def debug_embed(text: str, embedder: Optional[Union["TransformerEmbedder", "HashEmbeddingModel"]] = None) -> Dict[str, Any]:
+    """Embed *text* and return detailed diagnostics.
+
+    If *embedder* is ``None`` the default embedder from config is used.
+
+    Returns dict with ``text``, ``dim``, ``norm``, ``first_5``,
+    ``self_similarity``, and ``validation``.
+    """
+    if embedder is None:
+        embedder = get_embedder()
+
+    vec = embedder.embed_text(text)
+    val = validate_embedding(vec, label=text[:60])
+    self_sim = cosine_similarity(vec, vec)
+
+    return {
+        "text": text[:120],
+        "model": getattr(embedder, "model_key", "hash"),
+        "dim": len(vec),
+        "norm": val["norm"],
+        "first_5": vec[:5],
+        "self_similarity": self_sim,
+        "validation": val,
+    }

codegraph_cli/llm.py

@@ -51,7 +51,7 @@ class OllamaProvider(LLMProvider):
 
 
 class GroqProvider(LLMProvider):
-    """Groq cloud API provider using curl (workaround for Python urllib timeout)."""
+    """Groq cloud API provider."""
     
     def __init__(self, model: str, api_key: str):
         self.model = model
@@ -63,36 +63,24 @@ class GroqProvider(LLMProvider):
             return None
         
         try:
-            import subprocess
-            import tempfile
+            import requests
             
-            # Create JSON payload
-            payload = json.dumps({
-                "model": self.model,
-                "messages": [{"role": "user", "content": prompt}],
-                "temperature": 0.1,
-                "max_tokens": 1024,
-            })
-            
-            # Use curl (which works!) instead of Python HTTP libraries
-            result = subprocess.run(
-                [
-                    "curl", "-s", "-X", "POST",
-                    self.endpoint,
-                    "-H", "Content-Type: application/json",
-                    "-H", f"Authorization: Bearer {self.api_key}",
-                    "-d", payload,
-                    "--max-time", "15"
-                ],
-                capture_output=True,
-                text=True,
-                timeout=20
+            response = requests.post(
+                self.endpoint,
+                headers={
+                    "Content-Type": "application/json",
+                    "Authorization": f"Bearer {self.api_key}",
+                },
+                json={
+                    "model": self.model,
+                    "messages": [{"role": "user", "content": prompt}],
+                    "temperature": 0.1,
+                    "max_tokens": 1024,
+                },
+                timeout=20,
             )
-            
-            if result.returncode == 0 and result.stdout:
-                response = json.loads(result.stdout)
-                return response["choices"][0]["message"]["content"]
-            return None
+            response.raise_for_status()
+            return response.json()["choices"][0]["message"]["content"]
         except Exception:
             return None
 
@@ -372,33 +360,24 @@ class LocalLLM:
     
     def _chat_groq(self, messages: List[Dict], max_tokens: int, temperature: float) -> Optional[str]:
         """Chat completion for Groq."""
-        import subprocess
+        import requests
         
-        payload = json.dumps({
-            "model": self.model,
-            "messages": messages,
-            "temperature": temperature,
-            "max_tokens": max_tokens,
-        })
-        
-        result = subprocess.run(
-            [
-                "curl", "-s", "-X", "POST",
-                self.provider.endpoint,
-                "-H", "Content-Type: application/json",
-                "-H", f"Authorization: Bearer {self.api_key}",
-                "-d", payload,
-                "--max-time", "30"
-            ],
-            capture_output=True,
-            text=True,
-            timeout=35
+        response = requests.post(
+            self.provider.endpoint,
+            headers={
+                "Content-Type": "application/json",
+                "Authorization": f"Bearer {self.api_key}",
+            },
+            json={
+                "model": self.model,
+                "messages": messages,
+                "temperature": temperature,
+                "max_tokens": max_tokens,
+            },
+            timeout=35,
         )
-        
-        if result.returncode == 0 and result.stdout:
-            response = json.loads(result.stdout)
-            return response["choices"][0]["message"]["content"]
-        return None
+        response.raise_for_status()
+        return response.json()["choices"][0]["message"]["content"]
     
     def _chat_openai(self, messages: List[Dict], max_tokens: int, temperature: float) -> Optional[str]:
         """Chat completion for OpenAI."""
@@ -578,6 +557,7 @@ def create_crewai_llm(local_llm: LocalLLM):
         return LLM(
             model=f"ollama/{local_llm.model}",
             base_url=base_url,
+            max_tokens=4096,
         )
     
     elif provider == "groq":
@@ -585,6 +565,7 @@ def create_crewai_llm(local_llm: LocalLLM):
         return LLM(
             model=f"groq/{local_llm.model}",
             api_key=local_llm.api_key,
+            max_tokens=4096,
         )
     
     elif provider == "openai":
@@ -595,25 +576,29 @@ def create_crewai_llm(local_llm: LocalLLM):
             return LLM(
                 model=f"openrouter/{local_llm.model}",
                 api_key=local_llm.api_key,
+                max_tokens=4096,
             )
         elif local_llm.endpoint and local_llm.endpoint != "https://api.openai.com/v1/chat/completions":
             # Other custom endpoint - use base_url
             return LLM(
                 model=local_llm.model,
                 api_key=local_llm.api_key,
-                base_url=local_llm.endpoint.replace("/chat/completions", "")
+                base_url=local_llm.endpoint.replace("/chat/completions", ""),
+                max_tokens=4096,
             )
         else:
             # Standard OpenAI
             return LLM(
                 model=f"openai/{local_llm.model}",
                 api_key=local_llm.api_key,
+                max_tokens=4096,
             )
     
     elif provider == "anthropic":
         return LLM(
             model=f"anthropic/{local_llm.model}",
             api_key=local_llm.api_key,
+            max_tokens=4096,
         )
     
     elif provider == "gemini":
@@ -621,6 +606,7 @@ def create_crewai_llm(local_llm: LocalLLM):
         return LLM(
             model=f"gemini/{local_llm.model}",
             api_key=local_llm.api_key,
+            max_tokens=4096,
         )
     
     elif provider == "openrouter":
@@ -628,6 +614,7 @@ def create_crewai_llm(local_llm: LocalLLM):
         return LLM(
             model=f"openrouter/{local_llm.model}",
             api_key=local_llm.api_key,
+            max_tokens=4096,
         )
     
     else:

codegraph_cli/project_context.py

@@ -2,12 +2,15 @@
 
 from __future__ import annotations
 
+import logging
 from datetime import datetime
 from pathlib import Path
 from typing import Dict, List, Optional
 
 from .storage import GraphStore, ProjectManager
 
+logger = logging.getLogger(__name__)
+
 
 class ProjectContext:
     """Unified context for project with real file access and code graph."""
@@ -26,7 +29,10 @@ class ProjectContext:
         self.metadata = self.store.get_metadata()
         
         # Get source path from metadata
-        source_path_str = self.metadata.get("source_path")
+        source_path_str = (
+            self.metadata.get("source_path")
+            or self.metadata.get("project_root")
+        )
         if source_path_str:
             self.source_path = Path(source_path_str)
             if not self.source_path.exists():
@@ -143,8 +149,65 @@ class ProjectContext:
             full_path.parent.mkdir(parents=True, exist_ok=True)
         
         full_path.write_text(content, encoding="utf-8")
+        
+        # Auto-reindex the file so the code graph stays current
+        self._incremental_reindex(rel_path)
+        
         return True
     
+    def _incremental_reindex(self, rel_path: str) -> None:
+        """Parse and re-embed a single file into the code graph.
+
+        Called automatically after :meth:`write_file` so that newly
+        created or modified files are immediately searchable via RAG,
+        semantic search, and impact analysis — without requiring a
+        manual full re-index.
+
+        Silently skips if the source path is unavailable, the file
+        doesn't exist on disk, or the embedder cannot be loaded.
+        """
+        if not self.has_source_access:
+            return
+
+        file_path = self.source_path / rel_path
+        if not file_path.is_file():
+            return
+
+        try:
+            from .embeddings import get_embedder
+
+            embedder = get_embedder()
+            model_key = getattr(embedder, "model_key", "hash")
+
+            count = self.store.index_single_file(
+                file_path=file_path,
+                project_root=self.source_path,
+                embedder=embedder,
+                model_key=model_key,
+            )
+            if count:
+                logger.info(
+                    "Auto-indexed %d nodes for %s", count, rel_path,
+                )
+        except Exception as exc:
+            # Never let indexing failures break file operations
+            logger.debug("Incremental reindex failed for %s: %s", rel_path, exc)
+
+    def remove_from_index(self, rel_path: str) -> int:
+        """Remove a file's nodes and edges from the code graph.
+
+        Called after deleting a file so stale symbols don't appear in
+        search results.
+
+        Returns:
+            Number of nodes removed.
+        """
+        try:
+            return self.store.remove_nodes_for_file(rel_path)
+        except Exception as exc:
+            logger.debug("remove_from_index failed for %s: %s", rel_path, exc)
+            return 0
+    
     def file_exists(self, rel_path: str) -> bool:
         """Check if a file exists in the project.