codexa 0.4.0__py3-none-any.whl

semantic_code_intelligence/evolution/context_builder.py

@@ -0,0 +1,131 @@
+"""Context builder — constructs minimal LLM prompt context.
+
+The builder gathers *only* the information the LLM needs to generate a
+patch: the task description, the target file contents (or relevant
+excerpts), the git diff, and failing test output.  The assembled context
+is capped at a configurable token target (default 2 000 tokens ≈ 8 000
+chars) to keep LLM costs low.
+"""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+from semantic_code_intelligence.evolution.budget_guard import BudgetGuard
+from semantic_code_intelligence.evolution.commit_manager import CommitManager
+from semantic_code_intelligence.evolution.task_selector import EvolutionTask
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("evolution.context_builder")
+
+# Rough approximation: 1 token ≈ 4 characters
+_CHARS_PER_TOKEN = 4
+_DEFAULT_MAX_TOKENS = 2000
+
+# System prompt used for every evolution call
+SYSTEM_PROMPT = (
+    "You are a senior software engineer improving the CodexA codebase. "
+    "Your job is to make a SMALL, SAFE improvement. Rules:\n"
+    "- Change at most 3 files and 200 lines.\n"
+    "- Output ONLY a unified diff (no explanation before or after).\n"
+    "- Do NOT add new dependencies.\n"
+    "- Do NOT rewrite subsystems or change architecture.\n"
+    "- Preserve all existing tests.\n"
+    "- Use full type hints.\n"
+    "- Keep it simple."
+)
+
+
+class ContextBuilder:
+    """Assembles a minimal prompt context for the patch generator."""
+
+    def __init__(
+        self,
+        project_root: Path,
+        commit_manager: CommitManager,
+        max_context_tokens: int = _DEFAULT_MAX_TOKENS,
+    ) -> None:
+        self._root = project_root.resolve()
+        self._git = commit_manager
+        self._max_chars = max_context_tokens * _CHARS_PER_TOKEN
+
+    def build(self, task: EvolutionTask) -> str:
+        """Build the user-message portion of the LLM prompt.
+
+        Sections (included in priority order until budget exhausted):
+        1. Task description  (always included)
+        2. Target file contents (trimmed to budget)
+        3. Git diff           (if any)
+        4. Failing test output (if present in task context_hint)
+        """
+        parts: list[str] = []
+        budget = self._max_chars
+
+        # 1. Task description
+        header = self._task_section(task)
+        parts.append(header)
+        budget -= len(header)
+
+        # 2. Target file contents
+        for rel_path in task.target_files[:3]:
+            if budget <= 200:
+                break
+            section = self._file_section(rel_path, budget)
+            if section:
+                parts.append(section)
+                budget -= len(section)
+
+        # 3. Git diff
+        if budget > 200:
+            diff = self._git.git_diff()
+            if diff.strip():
+                diff_section = f"### Current git diff\n```diff\n{_truncate(diff, budget - 60)}\n```"
+                parts.append(diff_section)
+                budget -= len(diff_section)
+
+        # 4. Context hint (failing test output, etc.)
+        if budget > 200 and task.context_hint:
+            hint = f"### Additional context\n```\n{_truncate(task.context_hint, budget - 60)}\n```"
+            parts.append(hint)
+
+        return "\n\n".join(parts)
+
+    def estimate_tokens(self, text: str) -> int:
+        """Rough token estimate for a text string."""
+        return max(1, len(text) // _CHARS_PER_TOKEN)
+
+    # ------------------------------------------------------------------ #
+    # Section builders
+    # ------------------------------------------------------------------ #
+
+    @staticmethod
+    def _task_section(task: EvolutionTask) -> str:
+        lines = [
+            f"### Task: {task.category}",
+            task.description,
+        ]
+        if task.target_files:
+            lines.append(f"Target files: {', '.join(task.target_files)}")
+        lines.append(
+            "\nProduce a unified diff that implements this improvement."
+        )
+        return "\n".join(lines)
+
+    def _file_section(self, rel_path: str, budget: int) -> str | None:
+        """Read a target file and return a fenced code block."""
+        full = self._root / rel_path
+        if not full.exists():
+            return None
+        try:
+            content = full.read_text(encoding="utf-8", errors="replace")
+        except OSError:
+            return None
+        content = _truncate(content, budget - 80)
+        return f"### File: {rel_path}\n```python\n{content}\n```"
+
+
+def _truncate(text: str, max_chars: int) -> str:
+    """Truncate *text* to at most *max_chars*, appending '…' if trimmed."""
+    if len(text) <= max_chars:
+        return text
+    return text[:max_chars] + "\n... (truncated)"

semantic_code_intelligence/evolution/engine.py

@@ -0,0 +1,249 @@
+"""Evolution engine — orchestrates the self-improving development loop.
+
+Each iteration:
+1.  Run tests to get baseline.
+2.  Select a task (fix tests → type hints → error handling → dedup → optimise).
+3.  Build minimal LLM context.
+4.  Generate + apply patch via LLM.
+5.  Run tests again.
+6.  If tests pass → commit.  If tests fail → revert.
+7.  Repeat until budget exhausted.
+"""
+
+from __future__ import annotations
+
+import json
+import time
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from semantic_code_intelligence.evolution.budget_guard import BudgetGuard
+from semantic_code_intelligence.evolution.commit_manager import CommitManager
+from semantic_code_intelligence.evolution.context_builder import ContextBuilder
+from semantic_code_intelligence.evolution.patch_generator import PatchGenerator, PatchResult
+from semantic_code_intelligence.evolution.task_selector import EvolutionTask, TaskSelector
+from semantic_code_intelligence.evolution.test_runner import TestResult, TestRunner
+from semantic_code_intelligence.llm.provider import LLMProvider
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("evolution.engine")
+
+
+# ------------------------------------------------------------------ #
+# Result dataclasses
+# ------------------------------------------------------------------ #
+
+
+@dataclass
+class IterationRecord:
+    """Record of a single evolution iteration."""
+
+    iteration: int = 0
+    task_category: str = ""
+    task_description: str = ""
+    target_files: list[str] = field(default_factory=list)
+    patch_lines_changed: int = 0
+    tests_before: int = 0
+    tests_after: int = 0
+    committed: bool = False
+    commit_sha: str = ""
+    reverted: bool = False
+    error: str = ""
+    duration_seconds: float = 0.0
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialise the iteration record to a plain dictionary."""
+        return {
+            "iteration": self.iteration,
+            "task_category": self.task_category,
+            "task_description": self.task_description,
+            "target_files": self.target_files,
+            "patch_lines_changed": self.patch_lines_changed,
+            "tests_before": self.tests_before,
+            "tests_after": self.tests_after,
+            "committed": self.committed,
+            "commit_sha": self.commit_sha,
+            "reverted": self.reverted,
+            "error": self.error,
+            "duration_seconds": round(self.duration_seconds, 2),
+        }
+
+
+@dataclass
+class EvolutionResult:
+    """Aggregate result of an evolution run."""
+
+    iterations_completed: int = 0
+    commits: list[str] = field(default_factory=list)
+    reverts: int = 0
+    stop_reason: str = ""
+    budget_summary: dict[str, object] = field(default_factory=dict)
+    history: list[IterationRecord] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialise the evolution result to a plain dictionary."""
+        return {
+            "iterations_completed": self.iterations_completed,
+            "commits": self.commits,
+            "reverts": self.reverts,
+            "stop_reason": self.stop_reason,
+            "budget_summary": self.budget_summary,
+            "history": [r.to_dict() for r in self.history],
+        }
+
+
+# ------------------------------------------------------------------ #
+# Engine
+# ------------------------------------------------------------------ #
+
+
+class EvolutionEngine:
+    """Runs the self-improving development loop."""
+
+    def __init__(
+        self,
+        project_root: Path,
+        provider: LLMProvider,
+        budget: BudgetGuard,
+    ) -> None:
+        self._root = project_root.resolve()
+        self._provider = provider
+        self._budget = budget
+
+        # Build sub-components
+        self._test_runner = TestRunner(project_root=self._root)
+        self._commit_mgr = CommitManager(project_root=self._root)
+        self._ctx_builder = ContextBuilder(
+            project_root=self._root,
+            commit_manager=self._commit_mgr,
+        )
+        self._task_selector = TaskSelector(
+            project_root=self._root,
+            test_runner=self._test_runner,
+            commit_manager=self._commit_mgr,
+        )
+        self._patch_gen = PatchGenerator(
+            project_root=self._root,
+            provider=self._provider,
+            context_builder=self._ctx_builder,
+            budget=self._budget,
+        )
+
+    # ------------------------------------------------------------------ #
+    # Main loop
+    # ------------------------------------------------------------------ #
+
+    def run(self) -> EvolutionResult:
+        """Execute the evolution loop until the budget is exhausted."""
+        result = EvolutionResult()
+        self._budget.start()
+
+        # Baseline test run
+        last_test = self._test_runner.run()
+        logger.info("baseline tests: %s", last_test.summary_line())
+
+        iteration = 0
+        while self._budget.can_continue():
+            iteration += 1
+            iter_start = time.time()
+            record = IterationRecord(iteration=iteration, tests_before=last_test.passed)
+
+            try:
+                last_test = self._run_iteration(
+                    iteration, last_test, record, result,
+                )
+            except Exception as exc:
+                record.error = f"unexpected error: {exc}"
+                logger.error("iter %d: %s", iteration, record.error)
+
+            record.duration_seconds = time.time() - iter_start
+            result.history.append(record)
+            self._budget.record_iteration()
+
+        result.iterations_completed = iteration
+        result.stop_reason = self._budget.stop_reason() or "completed"
+        result.budget_summary = self._budget.summary()
+
+        # Persist history
+        self._write_history(result)
+        return result
+
+    # ------------------------------------------------------------------ #
+    # Single iteration
+    # ------------------------------------------------------------------ #
+
+    def _run_iteration(
+        self,
+        iteration: int,
+        last_test: TestResult,
+        record: IterationRecord,
+        result: EvolutionResult,
+    ) -> TestResult:
+        """Execute one evolution iteration.  Returns the latest TestResult."""
+        # 1. Select task
+        task = self._task_selector.select(last_test)
+        record.task_category = task.category
+        record.task_description = task.description
+        record.target_files = list(task.target_files)
+        logger.info("iter %d: %s — %s", iteration, task.category, task.description)
+
+        # 2. Generate + apply patch
+        patch_result = self._patch_gen.generate_and_apply(task)
+        record.patch_lines_changed = patch_result.lines_changed
+
+        if not patch_result.success:
+            record.error = patch_result.error
+            logger.warning("iter %d: patch failed — %s", iteration, patch_result.error)
+            return last_test
+
+        # 3. Re-run tests
+        new_test = self._test_runner.run()
+        record.tests_after = new_test.passed
+        logger.info(
+            "iter %d: tests %d -> %d",
+            iteration, last_test.passed, new_test.passed,
+        )
+
+        # 4. Decide: commit or revert
+        if new_test.return_code == 0 and new_test.passed >= last_test.passed:
+            self._commit_mgr.stage_files(patch_result.files_changed)
+            sha = self._commit_mgr.commit(
+                f"evolve: {task.category} — {task.description[:60]}"
+            )
+            record.committed = True
+            record.commit_sha = sha
+            result.commits.append(sha)
+            logger.info("iter %d: committed %s", iteration, sha)
+            return new_test
+
+        # Revert
+        self._commit_mgr.revert_files(patch_result.files_changed)
+        record.reverted = True
+        result.reverts += 1
+        logger.info("iter %d: reverted — tests regressed", iteration)
+        return last_test
+
+    # ------------------------------------------------------------------ #
+    # Persistence
+    # ------------------------------------------------------------------ #
+
+    def _write_history(self, result: EvolutionResult) -> None:
+        """Append the evolution run to ``.codexa/evolution_history.json``."""
+        history_dir = self._root / ".codexa"
+        history_dir.mkdir(parents=True, exist_ok=True)
+        history_file = history_dir / "evolution_history.json"
+
+        runs: list[dict[str, Any]] = []
+        if history_file.exists():
+            try:
+                runs = json.loads(history_file.read_text(encoding="utf-8"))
+            except (json.JSONDecodeError, OSError):
+                runs = []
+
+        runs.append(result.to_dict())
+        history_file.write_text(
+            json.dumps(runs, indent=2, default=str) + "\n",
+            encoding="utf-8",
+        )
+        logger.info("evolution history written to %s", history_file)

semantic_code_intelligence/evolution/patch_generator.py

@@ -0,0 +1,229 @@
+"""Patch generator — asks the LLM for a unified diff and applies it.
+
+The generator sends a minimal prompt (system + context) to the configured
+LLM provider and parses the response as a unified diff.  It then applies
+the diff to the working tree using ``git apply``.
+"""
+
+from __future__ import annotations
+
+import re
+import subprocess
+import tempfile
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from semantic_code_intelligence.evolution.budget_guard import BudgetGuard
+from semantic_code_intelligence.evolution.context_builder import SYSTEM_PROMPT, ContextBuilder
+from semantic_code_intelligence.evolution.task_selector import EvolutionTask
+from semantic_code_intelligence.llm.provider import LLMMessage, LLMProvider, LLMResponse, MessageRole
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("evolution.patch_generator")
+
+# Safety limits
+_MAX_FILES_CHANGED = 3
+_MAX_LINES_CHANGED = 200
+
+
+@dataclass
+class PatchResult:
+    """Result of a patch generation + apply attempt."""
+
+    success: bool = False
+    diff_text: str = ""
+    files_changed: list[str] = field(default_factory=list)
+    lines_changed: int = 0
+    llm_response: LLMResponse | None = None
+    error: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        """Serialise the patch result to a plain dictionary."""
+        return {
+            "success": self.success,
+            "files_changed": self.files_changed,
+            "lines_changed": self.lines_changed,
+            "error": self.error,
+        }
+
+
+class PatchGenerator:
+    """Generates and applies patches via LLM."""
+
+    def __init__(
+        self,
+        project_root: Path,
+        provider: LLMProvider,
+        context_builder: ContextBuilder,
+        budget: BudgetGuard,
+    ) -> None:
+        self._root = project_root.resolve()
+        self._provider = provider
+        self._ctx = context_builder
+        self._budget = budget
+
+    def generate_and_apply(self, task: EvolutionTask) -> PatchResult:
+        """Generate a patch and apply it to the working tree.
+
+        Steps:
+        1.  Build minimal context prompt.
+        2.  Call LLM.
+        3.  Parse unified diff from response.
+        4.  Validate safety limits.
+        5.  Apply via ``git apply``.
+        """
+        # 1. Build context
+        user_msg = self._ctx.build(task)
+        prompt_tokens = self._ctx.estimate_tokens(SYSTEM_PROMPT + user_msg)
+
+        if prompt_tokens > self._budget.tokens_remaining:
+            return PatchResult(
+                error=f"Prompt ({prompt_tokens} tokens) exceeds remaining budget "
+                      f"({self._budget.tokens_remaining} tokens)."
+            )
+
+        # 2. Call LLM
+        messages = [
+            LLMMessage(role=MessageRole.SYSTEM, content=SYSTEM_PROMPT),
+            LLMMessage(role=MessageRole.USER, content=user_msg),
+        ]
+        try:
+            response = self._provider.chat(messages, temperature=0.2, max_tokens=2048)
+        except Exception as exc:
+            return PatchResult(error=f"LLM call failed: {exc}")
+
+        total_tokens = response.usage.get("total_tokens", prompt_tokens + len(response.content) // 4)
+        self._budget.record_tokens(total_tokens)
+
+        # 3. Parse diff
+        diff_text = _extract_diff(response.content)
+        if not diff_text:
+            return PatchResult(
+                llm_response=response,
+                error="LLM response did not contain a valid unified diff.",
+            )
+
+        # 4. Validate safety limits
+        files = _diff_files(diff_text)
+        lines = _diff_line_count(diff_text)
+
+        if len(files) > _MAX_FILES_CHANGED:
+            return PatchResult(
+                diff_text=diff_text,
+                files_changed=files,
+                lines_changed=lines,
+                error=f"Patch touches {len(files)} files (max {_MAX_FILES_CHANGED}).",
+            )
+        if lines > _MAX_LINES_CHANGED:
+            return PatchResult(
+                diff_text=diff_text,
+                files_changed=files,
+                lines_changed=lines,
+                error=f"Patch changes {lines} lines (max {_MAX_LINES_CHANGED}).",
+            )
+
+        # 5. Apply
+        ok, apply_err = _apply_diff(diff_text, self._root)
+        if not ok:
+            return PatchResult(
+                diff_text=diff_text,
+                files_changed=files,
+                lines_changed=lines,
+                error=f"git apply failed: {apply_err}",
+            )
+
+        return PatchResult(
+            success=True,
+            diff_text=diff_text,
+            files_changed=files,
+            lines_changed=lines,
+            llm_response=response,
+        )
+
+
+# ------------------------------------------------------------------ #
+# Diff parsing helpers
+# ------------------------------------------------------------------ #
+
+def _extract_diff(text: str) -> str:
+    """Extract a unified diff block from LLM output.
+
+    Looks for ``--- a/`` or a fenced code block containing diff content.
+    """
+    # Try fenced code block first
+    m = re.search(r"```(?:diff)?\s*\n(.*?)```", text, re.DOTALL)
+    if m:
+        candidate = m.group(1).strip()
+        if "--- " in candidate or "+++ " in candidate:
+            return candidate
+
+    # Try raw diff
+    lines = text.splitlines()
+    diff_lines: list[str] = []
+    in_diff = False
+    for line in lines:
+        if line.startswith("--- ") or line.startswith("+++ ") or line.startswith("@@ "):
+            in_diff = True
+        if in_diff:
+            diff_lines.append(line)
+            # Stop on blank line after diff section
+            if not line.strip() and diff_lines and diff_lines[-2].strip():
+                continue
+    return "\n".join(diff_lines).strip()
+
+
+def _diff_files(diff_text: str) -> list[str]:
+    """Extract file paths changed in a unified diff."""
+    files: list[str] = []
+    for line in diff_text.splitlines():
+        if line.startswith("+++ "):
+            path = line[4:].strip()
+            # Strip "b/" prefix
+            if path.startswith("b/"):
+                path = path[2:]
+            if path and path != "/dev/null" and path not in files:
+                files.append(path)
+    return files
+
+
+def _diff_line_count(diff_text: str) -> int:
+    """Count the number of added + removed lines in a diff."""
+    count = 0
+    for line in diff_text.splitlines():
+        if line.startswith("+") and not line.startswith("+++"):
+            count += 1
+        elif line.startswith("-") and not line.startswith("---"):
+            count += 1
+    return count
+
+
+def _apply_diff(diff_text: str, cwd: Path) -> tuple[bool, str]:
+    """Apply a unified diff via ``git apply``.  Returns (success, error)."""
+    with tempfile.NamedTemporaryFile(
+        mode="w", suffix=".patch", delete=False, encoding="utf-8"
+    ) as f:
+        f.write(diff_text)
+        patch_path = f.name
+
+    try:
+        proc = subprocess.run(
+            ["git", "apply", "--check", patch_path],
+            capture_output=True,
+            text=True,
+            cwd=str(cwd),
+        )
+        if proc.returncode != 0:
+            return False, proc.stderr.strip()
+
+        proc = subprocess.run(
+            ["git", "apply", patch_path],
+            capture_output=True,
+            text=True,
+            cwd=str(cwd),
+        )
+        if proc.returncode != 0:
+            return False, proc.stderr.strip()
+        return True, ""
+    finally:
+        Path(patch_path).unlink(missing_ok=True)