devagent-cli 3.2.1__py3-none-any.whl

devagent/app/llm.py

@@ -0,0 +1,83 @@
+"""
+LLM integration layer — talks to Ollama via the Python SDK.
+All prompts are kept SHORT to respect low-VRAM constraints.
+
+Supports:
+  - Configurable model + inference options
+  - Latency tracking
+  - Graceful fallback on errors
+"""
+
+from __future__ import annotations
+
+import time
+import ollama
+
+from devagent.utils.config import MODELS, DEFAULT_INFERENCE_OPTIONS
+
+# ── Configuration (mutable at runtime via CLI) ───────────────────────────────
+MODEL = MODELS["primary"]
+OPTIONS = DEFAULT_INFERENCE_OPTIONS.copy()
+
+
+def set_model(model: str) -> None:
+    """Override the active model."""
+    global MODEL
+    MODEL = model
+
+
+def set_options(options: dict) -> None:
+    """Override inference options."""
+    global OPTIONS
+    OPTIONS = {**DEFAULT_INFERENCE_OPTIONS, **options}
+
+
+def query(prompt: str, *, system: str = "", model: str | None = None) -> str:
+    """Send a prompt to Ollama and return the response text.
+
+    Falls back to empty string on error so the agent loop can continue.
+    Returns the response text.
+    """
+    use_model = model or MODEL
+    messages: list[dict[str, str]] = []
+    if system:
+        messages.append({"role": "system", "content": system})
+    messages.append({"role": "user", "content": prompt})
+
+    try:
+        t0 = time.time()
+        response = ollama.chat(
+            model=use_model,
+            messages=messages,
+            options=OPTIONS,
+        )
+        elapsed = time.time() - t0
+        text = response["message"]["content"].strip()
+
+        # Track latency for metrics (stored globally for metrics collector)
+        query._last_latency = elapsed
+        query._last_prompt_chars = len(prompt)
+        query._last_response_chars = len(text)
+
+        return text
+    except Exception as exc:  # noqa: BLE001
+        print(f"[LLM ERROR] {exc}")
+        query._last_latency = 0
+        query._last_prompt_chars = len(prompt)
+        query._last_response_chars = 0
+        return ""
+
+# Initialize tracking attributes
+query._last_latency = 0.0
+query._last_prompt_chars = 0
+query._last_response_chars = 0
+
+
+def query_with_context(prompt: str, context: str, *, model: str | None = None) -> str:
+    """Query with additional context injected into the system message."""
+    system = (
+        "You are a senior Python developer. "
+        "Be concise. Output ONLY what is asked — no markdown fences unless requested.\n\n"
+        f"CONTEXT:\n{context[:2000]}"
+    )
+    return query(prompt, system=system, model=model)

devagent/app/memory.py

@@ -0,0 +1,309 @@
+"""
+Memory Layer — Short-term runtime state + Long-term FAISS-backed semantic retrieval.
+
+Implements:
+  - Semantic code chunking
+  - Sentence-transformer embeddings
+  - FAISS vector index for Top-K retrieval
+  - Compressed memory summaries
+  - Context pruning for low-VRAM operation
+
+Falls back gracefully if sentence-transformers or faiss are not installed.
+"""
+
+from __future__ import annotations
+
+import os
+import hashlib
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from devagent.utils.config import (
+    SUPPORTED_EXTENSIONS,
+    IGNORE_DIRS,
+    MAX_FILE_SIZE_BYTES,
+    MAX_CHUNK_CHARS,
+    TOP_K_RETRIEVAL,
+)
+
+# ── Lazy imports for optional heavy deps ──────────────────────────────────────
+
+_FAISS_AVAILABLE = False
+_TRANSFORMERS_AVAILABLE = False
+_np = None
+_faiss = None
+_SentenceTransformer = None
+
+
+def _load_deps() -> None:
+    """Lazy-load heavy dependencies only when needed."""
+    global _FAISS_AVAILABLE, _TRANSFORMERS_AVAILABLE, _np, _faiss, _SentenceTransformer
+    try:
+        import numpy as np
+        _np = np
+    except ImportError:
+        return
+
+    try:
+        import faiss
+        _faiss = faiss
+        _FAISS_AVAILABLE = True
+    except ImportError:
+        pass
+
+    try:
+        from sentence_transformers import SentenceTransformer
+        _SentenceTransformer = SentenceTransformer
+        _TRANSFORMERS_AVAILABLE = True
+    except ImportError:
+        pass
+
+
+# ── Code Chunking ─────────────────────────────────────────────────────────────
+
+@dataclass
+class CodeChunk:
+    """A chunk of source code with metadata."""
+    file_path: str
+    start_line: int
+    end_line: int
+    content: str
+    language: str = "python"
+    chunk_hash: str = ""
+
+    def __post_init__(self) -> None:
+        if not self.chunk_hash:
+            self.chunk_hash = hashlib.md5(
+                self.content.encode("utf-8", errors="replace")
+            ).hexdigest()[:12]
+
+
+def chunk_file(file_path: str, max_chars: int = MAX_CHUNK_CHARS) -> list[CodeChunk]:
+    """Split a source file into semantic chunks.
+
+    Chunks on function/class boundaries when possible, falls back to
+    fixed-size splitting.
+    """
+    try:
+        content = Path(file_path).read_text(encoding="utf-8", errors="replace")
+    except Exception:
+        return []
+
+    if len(content) > MAX_FILE_SIZE_BYTES:
+        return []  # Skip very large files
+
+    ext = Path(file_path).suffix
+    lines = content.splitlines(keepends=True)
+    chunks: list[CodeChunk] = []
+    current_lines: list[str] = []
+    current_start = 1
+
+    def _flush(end: int) -> None:
+        text = "".join(current_lines).strip()
+        if text:
+            chunks.append(CodeChunk(
+                file_path=file_path,
+                start_line=current_start,
+                end_line=end,
+                content=text[:max_chars],
+                language=ext.lstrip("."),
+            ))
+
+    for i, line in enumerate(lines, start=1):
+        stripped = line.strip()
+
+        # Split on function/class boundaries (Python-aware)
+        is_boundary = (
+            stripped.startswith("def ") or
+            stripped.startswith("class ") or
+            stripped.startswith("async def ")
+        )
+
+        if is_boundary and current_lines:
+            _flush(i - 1)
+            current_lines = [line]
+            current_start = i
+        else:
+            current_lines.append(line)
+
+        # Force split if chunk gets too large
+        if len("".join(current_lines)) > max_chars:
+            _flush(i)
+            current_lines = []
+            current_start = i + 1
+
+    # Flush remaining
+    if current_lines:
+        _flush(len(lines))
+
+    return chunks
+
+
+def chunk_project(project_root: str) -> list[CodeChunk]:
+    """Recursively chunk all source files in a project."""
+    all_chunks: list[CodeChunk] = []
+    for root, dirs, files in os.walk(project_root):
+        # Skip ignored directories
+        dirs[:] = [d for d in dirs if d not in IGNORE_DIRS]
+        for f in files:
+            if Path(f).suffix in SUPPORTED_EXTENSIONS:
+                full_path = os.path.join(root, f)
+                all_chunks.extend(chunk_file(full_path))
+    return all_chunks
+
+
+# ── FAISS Vector Store ────────────────────────────────────────────────────────
+
+class SemanticIndex:
+    """FAISS-backed semantic search index for code chunks.
+
+    Falls back to keyword matching if FAISS or sentence-transformers
+    are not available.
+    """
+
+    EMBED_MODEL = "all-MiniLM-L6-v2"  # 80 MB, fast, good quality
+
+    def __init__(self) -> None:
+        _load_deps()
+        self.chunks: list[CodeChunk] = []
+        self._index = None
+        self._embedder = None
+        self._dimension = 0
+        self._ready = False
+
+    def build(self, chunks: list[CodeChunk]) -> bool:
+        """Build the FAISS index from code chunks.
+
+        Returns True if semantic index was built, False if falling back.
+        """
+        self.chunks = chunks
+        if not chunks:
+            return False
+
+        if not (_FAISS_AVAILABLE and _TRANSFORMERS_AVAILABLE):
+            print("[MEMORY] FAISS/sentence-transformers not available — using keyword fallback")
+            return False
+
+        try:
+            print(f"[MEMORY] Building semantic index over {len(chunks)} chunks...")
+            self._embedder = _SentenceTransformer(self.EMBED_MODEL)
+            texts = [c.content for c in chunks]
+            embeddings = self._embedder.encode(texts, show_progress_bar=False)
+            embeddings = _np.array(embeddings, dtype="float32")
+
+            self._dimension = embeddings.shape[1]
+            self._index = _faiss.IndexFlatIP(self._dimension)  # Inner product
+            _faiss.normalize_L2(embeddings)
+            self._index.add(embeddings)
+
+            self._ready = True
+            print(f"[MEMORY] Semantic index ready ({self._dimension}d, {len(chunks)} chunks)")
+            return True
+        except Exception as exc:
+            print(f"[MEMORY] Index build failed: {exc}")
+            self._ready = False
+            return False
+
+    def search(self, query: str, top_k: int = TOP_K_RETRIEVAL) -> list[CodeChunk]:
+        """Retrieve the most relevant chunks for a query."""
+        if not self.chunks:
+            return []
+
+        if self._ready and self._embedder and self._index:
+            return self._semantic_search(query, top_k)
+        else:
+            return self._keyword_search(query, top_k)
+
+    def _semantic_search(self, query: str, top_k: int) -> list[CodeChunk]:
+        """FAISS-powered semantic search."""
+        try:
+            q_embed = self._embedder.encode([query])
+            q_embed = _np.array(q_embed, dtype="float32")
+            _faiss.normalize_L2(q_embed)
+
+            k = min(top_k, len(self.chunks))
+            scores, indices = self._index.search(q_embed, k)
+
+            results = []
+            for idx in indices[0]:
+                if 0 <= idx < len(self.chunks):
+                    results.append(self.chunks[idx])
+            return results
+        except Exception as exc:
+            print(f"[MEMORY] Semantic search failed: {exc}")
+            return self._keyword_search(query, top_k)
+
+    def _keyword_search(self, query: str, top_k: int) -> list[CodeChunk]:
+        """Fallback keyword-based search."""
+        query_lower = query.lower()
+        scored = []
+        for chunk in self.chunks:
+            content_lower = chunk.content.lower()
+            score = content_lower.count(query_lower)
+            # Boost exact matches in function/class names
+            for line in chunk.content.splitlines()[:3]:
+                if query_lower in line.lower():
+                    score += 5
+            if score > 0:
+                scored.append((score, chunk))
+
+        scored.sort(key=lambda x: x[0], reverse=True)
+        return [c for _, c in scored[:top_k]]
+
+
+# ── Working Memory (Short-Term) ──────────────────────────────────────────────
+
+@dataclass
+class WorkingMemory:
+    """Short-term memory for the current agent run.
+
+    Stores retrieved chunks, compressed summaries, and recent
+    observations for context assembly.
+    """
+
+    retrieved_chunks: list[CodeChunk] = field(default_factory=list)
+    summaries: list[str] = field(default_factory=list)
+    observations: list[str] = field(default_factory=list)
+    thoughts: list[str] = field(default_factory=list)
+
+    def add_retrieval(self, chunks: list[CodeChunk]) -> None:
+        """Add retrieved chunks, deduplicating by hash."""
+        seen = {c.chunk_hash for c in self.retrieved_chunks}
+        for c in chunks:
+            if c.chunk_hash not in seen:
+                self.retrieved_chunks.append(c)
+                seen.add(c.chunk_hash)
+
+    def add_chunk(self, chunk: CodeChunk) -> None:
+        """Add a single chunk to memory."""
+        self.add_retrieval([chunk])
+
+    def get_context(self, max_chars: int = 3000) -> str:
+        """Assemble compressed context from retrieved chunks."""
+        parts: list[str] = []
+        total = 0
+
+        for chunk in self.retrieved_chunks:
+            header = f"# {chunk.file_path} (L{chunk.start_line}-{chunk.end_line})"
+            block = f"{header}\n{chunk.content}\n"
+            if total + len(block) > max_chars:
+                break
+            parts.append(block)
+            total += len(block)
+
+        return "\n".join(parts)
+
+    def add_summary(self, summary: str) -> None:
+        """Store a compressed summary."""
+        self.summaries.append(summary[:500])
+        # Keep only last 10 summaries
+        if len(self.summaries) > 10:
+            self.summaries = self.summaries[-10:]
+
+    def clear(self) -> None:
+        """Reset working memory."""
+        self.retrieved_chunks.clear()
+        self.summaries.clear()
+        self.observations.clear()
+        self.thoughts.clear()

devagent/app/patcher.py

@@ -0,0 +1,83 @@
+"""
+Patch Engine — generates and applies line-level diffs instead of rewriting full files.
+
+Supports:
+  - Unified diff generation
+  - Patch application with context validation
+  - Minimal change preservation
+"""
+
+from __future__ import annotations
+
+import difflib
+import os
+from pathlib import Path
+from typing import Any
+
+
+def generate_diff(original: str, modified: str, file_path: str = "file.py") -> str:
+    """Generate a unified diff between original and modified content."""
+    orig_lines = original.splitlines(keepends=True)
+    mod_lines = modified.splitlines(keepends=True)
+
+    diff = difflib.unified_diff(
+        orig_lines, mod_lines,
+        fromfile=f"a/{file_path}",
+        tofile=f"b/{file_path}",
+        lineterm="",
+    )
+    return "".join(diff)
+
+
+def apply_patch(file_path: str, original_content: str, patched_content: str) -> dict[str, Any]:
+    """Apply a patch to a file with validation.
+
+    Returns a result dict with status, diff, and stats.
+    """
+    result: dict[str, Any] = {
+        "status": "error",
+        "file": file_path,
+        "diff": "",
+        "lines_changed": 0,
+        "lines_added": 0,
+        "lines_removed": 0,
+    }
+
+    if not patched_content.strip():
+        result["error"] = "Empty patch content"
+        return result
+
+    # Generate diff for logging
+    diff_text = generate_diff(original_content, patched_content, file_path)
+    result["diff"] = diff_text
+
+    # Count changes
+    for line in diff_text.splitlines():
+        if line.startswith("+") and not line.startswith("+++"):
+            result["lines_added"] += 1
+        elif line.startswith("-") and not line.startswith("---"):
+            result["lines_removed"] += 1
+    result["lines_changed"] = result["lines_added"] + result["lines_removed"]
+
+    # Write the patched file
+    try:
+        p = Path(file_path)
+        p.parent.mkdir(parents=True, exist_ok=True)
+        p.write_text(patched_content, encoding="utf-8")
+        result["status"] = "success"
+    except Exception as exc:
+        result["status"] = "error"
+        result["error"] = str(exc)
+
+    return result
+
+
+def format_diff_summary(patch_result: dict[str, Any]) -> str:
+    """Format a human-readable patch summary."""
+    if patch_result["status"] != "success":
+        return f"[PATCH ERROR] {patch_result.get('error', 'Unknown error')}"
+
+    return (
+        f"[PATCH] {patch_result['file']}: "
+        f"+{patch_result['lines_added']} / -{patch_result['lines_removed']} lines"
+    )

devagent/app/planner.py

@@ -0,0 +1,76 @@
+"""
+Planner Layer — interprets the task and generates a short action plan.
+
+The planner is NOT the coder. It decides:
+  1. Which files are likely relevant
+  2. Which tools to use first
+  3. A short execution strategy
+"""
+
+from __future__ import annotations
+
+import re
+from typing import Any
+
+from devagent.app.llm import query
+
+
+PLAN_PROMPT = """\
+You are a coding task planner. Given a task and project files, create a SHORT plan.
+
+TASK: {task}
+
+PROJECT FILES:
+{file_list}
+
+Create a plan with exactly 3-5 steps. Each step should be ONE action.
+Available actions: search_code, semantic_search, read_file, write_patch, run_tests, lint_code, git_diff
+
+Reply in this EXACT format:
+LIKELY_FILES: file1.py, file2.py
+PLAN:
+1. <action>: <target>
+2. <action>: <target>
+3. <action>: <target>
+"""
+
+
+def generate_plan(task: str, file_list: list[str]) -> dict[str, Any]:
+    """Generate an execution plan for the given task."""
+    files_str = "\n".join(f"  - {f}" for f in file_list[:30])
+    prompt = PLAN_PROMPT.format(task=task, file_list=files_str)
+    response = query(prompt)
+
+    if not response:
+        return _fallback_plan(task, file_list)
+    return _parse_plan(response, task, file_list)
+
+
+def _parse_plan(response: str, task: str, file_list: list[str]) -> dict[str, Any]:
+    """Parse the LLM's plan response."""
+    result: dict[str, Any] = {"likely_files": [], "steps": [], "raw_plan": response}
+
+    files_match = re.search(r"LIKELY_FILES:\s*(.+)", response, re.IGNORECASE)
+    if files_match:
+        result["likely_files"] = [f.strip() for f in files_match.group(1).split(",") if f.strip()]
+
+    step_pattern = re.compile(r"\d+\.\s*(\w+):\s*(.+)")
+    for match in step_pattern.finditer(response):
+        result["steps"].append((match.group(1).strip().lower(), match.group(2).strip()))
+
+    if not result["steps"]:
+        return _fallback_plan(task, file_list)
+    return result
+
+
+def _fallback_plan(task: str, file_list: list[str]) -> dict[str, Any]:
+    """Generate a sensible fallback plan from the task description."""
+    likely_files = [m.group(1) for m in re.finditer(r"(\w+\.py)", task)]
+    steps = []
+    if likely_files:
+        steps.append(("read_file", likely_files[0]))
+    else:
+        keywords = [w for w in task.lower().split() if len(w) > 3 and w not in {"the", "that", "this", "with", "from"}]
+        steps.append(("search_code", keywords[0] if keywords else "def"))
+    steps.append(("run_tests", "."))
+    return {"likely_files": likely_files, "steps": steps, "raw_plan": "Fallback plan."}

devagent/app/reviewer.py

@@ -0,0 +1,65 @@
+"""
+Self-review module.
+The LLM critiques its own code fix and returns APPROVED or REVISE + reason.
+"""
+
+from __future__ import annotations
+
+from devagent.app.llm import query
+
+REVIEW_SYSTEM = (
+    "You are a strict code reviewer. "
+    "Review the proposed code fix for correctness, edge cases, and syntax. "
+    "CRITICAL RULES:\n"
+    "1. If the code has a SyntaxError or unclosed blocks, return REVISE.\n"
+    "2. If the proposed fix adds unnecessary tests, example usages, or 'if __name__ == \"__main__\":' blocks, return REVISE.\n"
+    "3. If the fix is correct and minimal, return APPROVED.\n\n"
+    "Respond with EXACTLY one line:\n"
+    "  APPROVED\n"
+    "or\n"
+    "  REVISE: <one-line reason>\n"
+    "Do NOT output anything else."
+)
+
+REVISE_SYSTEM = (
+    "You are a senior Python developer. "
+    "Fix the code based on the review feedback. "
+    "Output ONLY the corrected Python code — no explanations, no markdown fences."
+)
+
+
+def review_code(original_code: str, fixed_code: str, task: str) -> tuple[bool, str]:
+    """Review a code fix.
+
+    Returns:
+        (approved: bool, review_text: str)
+    """
+    prompt = (
+        f"TASK: {task}\n\n"
+        f"ORIGINAL CODE:\n{original_code[:1500]}\n\n"
+        f"PROPOSED FIX:\n{fixed_code[:1500]}\n\n"
+        "Is the fix correct? Reply APPROVED or REVISE: <reason>"
+    )
+    response = query(prompt, system=REVIEW_SYSTEM)
+
+    if not response:
+        # LLM failure → cautiously approve to avoid infinite loop
+        return True, "APPROVED (LLM unavailable — auto-approved)"
+
+    approved = response.strip().upper().startswith("APPROVED")
+    return approved, response.strip()
+
+
+def revise_code(code: str, review_feedback: str, task: str) -> str:
+    """Ask the LLM to revise code based on review feedback.
+
+    Returns the revised code string.
+    """
+    prompt = (
+        f"TASK: {task}\n\n"
+        f"CODE:\n{code[:1500]}\n\n"
+        f"REVIEW FEEDBACK: {review_feedback}\n\n"
+        "Fix the code. Output ONLY the corrected Python code."
+    )
+    response = query(prompt, system=REVISE_SYSTEM)
+    return response if response else code  # fallback to original on failure