codexa 0.4.0__py3-none-any.whl

semantic_code_intelligence/evolution/task_selector.py

@@ -0,0 +1,214 @@
+"""Task selector — chooses the next small improvement task.
+
+Analyses the current repository state (git diff, failing tests, code
+quality signals) and picks a single, well-scoped task for the LLM to
+implement.  Every task targets **≤3 files** and **≤200 lines changed**.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from semantic_code_intelligence.evolution.commit_manager import CommitManager
+from semantic_code_intelligence.evolution.test_runner import TestResult, TestRunner
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("evolution.task_selector")
+
+# Priority-ordered task categories
+TASK_FIX_TESTS = "fix_failing_tests"
+TASK_TYPE_HINTS = "add_missing_type_hints"
+TASK_ERROR_HANDLING = "improve_error_handling"
+TASK_REDUCE_DUPLICATION = "reduce_duplication"
+TASK_SMALL_OPTIMISATION = "small_performance_optimisation"
+
+TASK_PRIORITIES: list[str] = [
+    TASK_FIX_TESTS,
+    TASK_TYPE_HINTS,
+    TASK_ERROR_HANDLING,
+    TASK_REDUCE_DUPLICATION,
+    TASK_SMALL_OPTIMISATION,
+]
+
+
+@dataclass
+class EvolutionTask:
+    """A single, well-scoped improvement task."""
+
+    category: str
+    description: str
+    target_files: list[str] = field(default_factory=list)
+    context_hint: str = ""
+
+    def to_dict(self) -> dict[str, object]:
+        """Serialise the task to a plain dictionary."""
+        return {
+            "category": self.category,
+            "description": self.description,
+            "target_files": self.target_files,
+            "context_hint": self.context_hint,
+        }
+
+
+class TaskSelector:
+    """Selects the next evolution task based on repo state."""
+
+    def __init__(
+        self,
+        project_root: Path,
+        test_runner: TestRunner,
+        commit_manager: CommitManager,
+    ) -> None:
+        self._root = project_root.resolve()
+        self._runner = test_runner
+        self._git = commit_manager
+
+    def select(self, last_test_result: TestResult | None = None) -> EvolutionTask:
+        """Choose the highest-priority actionable task.
+
+        1. If tests are failing → fix them
+        2. Else scan for missing type hints
+        3. Else scan for bare excepts / weak error handling
+        4. Else look for obvious duplication
+        5. Fallback: small quality improvement
+        """
+        # Priority 1: fix failing tests
+        if last_test_result and not last_test_result.passed:
+            return self._task_from_failures(last_test_result)
+
+        # Priority 2–5: static analysis of source files
+        src_dir = self._root / "semantic_code_intelligence"
+        py_files = self._collect_py_files(src_dir)
+
+        task = self._find_type_hint_task(py_files)
+        if task:
+            return task
+
+        task = self._find_error_handling_task(py_files)
+        if task:
+            return task
+
+        task = self._find_duplication_task(py_files)
+        if task:
+            return task
+
+        # Fallback
+        return EvolutionTask(
+            category=TASK_SMALL_OPTIMISATION,
+            description="Look for a small quality or performance improvement in the codebase.",
+            target_files=[],
+            context_hint="Focus on hot-path functions or frequently used utilities.",
+        )
+
+    # ------------------------------------------------------------------ #
+    # Task builders
+    # ------------------------------------------------------------------ #
+
+    def _task_from_failures(self, result: TestResult) -> EvolutionTask:
+        """Extract a fix-tests task from failing test output."""
+        # Pull failing file hints from the output (pytest --tb=line gives file:line)
+        failing_files: list[str] = []
+        for line in result.output.splitlines():
+            stripped = line.strip()
+            if stripped.startswith("FAILED ") or "::" in stripped:
+                parts = stripped.split("::")
+                if parts:
+                    fpath = parts[0].replace("FAILED ", "").strip()
+                    if fpath.endswith(".py") and fpath not in failing_files:
+                        failing_files.append(fpath)
+        return EvolutionTask(
+            category=TASK_FIX_TESTS,
+            description=f"Fix {result.failures} failing test(s).",
+            target_files=failing_files[:3],
+            context_hint=_last_n_lines(result.output, 40),
+        )
+
+    def _find_type_hint_task(self, files: list[Path]) -> EvolutionTask | None:
+        """Find a source file with functions lacking return type annotations."""
+        import re
+        pattern = re.compile(r"^\s*def\s+\w+\([^)]*\)\s*:", re.MULTILINE)
+        typed = re.compile(r"^\s*def\s+\w+\([^)]*\)\s*->\s*", re.MULTILINE)
+
+        for fpath in files:
+            try:
+                text = fpath.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                continue
+            all_defs = pattern.findall(text)
+            typed_defs = typed.findall(text)
+            missing = len(all_defs) - len(typed_defs)
+            if missing >= 2:
+                rel = str(fpath.relative_to(self._root))
+                return EvolutionTask(
+                    category=TASK_TYPE_HINTS,
+                    description=f"Add return type hints to {missing} function(s) in {rel}.",
+                    target_files=[rel],
+                    context_hint=f"File has {len(all_defs)} defs, {len(typed_defs)} typed.",
+                )
+        return None
+
+    def _find_error_handling_task(self, files: list[Path]) -> EvolutionTask | None:
+        """Find a file with bare ``except:`` or ``except Exception:`` blocks."""
+        for fpath in files:
+            try:
+                text = fpath.read_text(encoding="utf-8", errors="replace")
+            except OSError:
+                continue
+            if "\nexcept:" in text or "\nexcept Exception:" in text:
+                rel = str(fpath.relative_to(self._root))
+                return EvolutionTask(
+                    category=TASK_ERROR_HANDLING,
+                    description=f"Replace bare/broad except blocks with specific exceptions in {rel}.",
+                    target_files=[rel],
+                    context_hint="Catch only the exceptions that can actually occur.",
+                )
+        return None
+
+    def _find_duplication_task(self, files: list[Path]) -> EvolutionTask | None:
+        """Very lightweight duplication detector — looks for repeated blocks."""
+        # Simplified: look for files > 300 lines with repeated 5-line blocks
+        for fpath in files:
+            try:
+                lines = fpath.read_text(encoding="utf-8", errors="replace").splitlines()
+            except OSError:
+                continue
+            if len(lines) < 300:
+                continue
+            blocks: dict[str, int] = {}
+            for i in range(len(lines) - 4):
+                block = "\n".join(lines[i : i + 5]).strip()
+                if len(block) > 60:
+                    blocks[block] = blocks.get(block, 0) + 1
+            dups = sum(1 for v in blocks.values() if v >= 2)
+            if dups >= 2:
+                rel = str(fpath.relative_to(self._root))
+                return EvolutionTask(
+                    category=TASK_REDUCE_DUPLICATION,
+                    description=f"Extract duplicated logic into helper functions in {rel}.",
+                    target_files=[rel],
+                    context_hint=f"Found {dups} repeated 5-line blocks.",
+                )
+        return None
+
+    # ------------------------------------------------------------------ #
+    # Helpers
+    # ------------------------------------------------------------------ #
+
+    def _collect_py_files(self, src_dir: Path) -> list[Path]:
+        """Collect .py source files, excluding tests and __pycache__."""
+        results: list[Path] = []
+        if not src_dir.exists():
+            return results
+        for fpath in sorted(src_dir.rglob("*.py")):
+            rel = str(fpath.relative_to(self._root))
+            if "tests" in rel or "__pycache__" in rel:
+                continue
+            results.append(fpath)
+        return results
+
+
+def _last_n_lines(text: str, n: int) -> str:
+    """Return the last *n* non-empty lines of *text*."""
+    lines = [l for l in text.splitlines() if l.strip()]
+    return "\n".join(lines[-n:])

semantic_code_intelligence/evolution/test_runner.py

@@ -0,0 +1,111 @@
+"""Test runner — executes pytest and returns structured results.
+
+Runs ``pytest`` as a subprocess to avoid polluting the current process
+with imported test modules.  Returns a structured ``TestResult`` that
+the engine can use to decide whether to commit or revert.
+"""
+
+from __future__ import annotations
+
+import subprocess
+import sys
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("evolution.test_runner")
+
+
+@dataclass
+class TestResult:
+    """Structured test-run result."""
+
+    __test__ = False  # prevent pytest collection
+
+    passed: bool = False
+    total: int = 0
+    failures: int = 0
+    errors: int = 0
+    output: str = ""
+    return_code: int = -1
+
+    def summary_line(self) -> str:
+        status = "PASS" if self.passed else "FAIL"
+        return f"[{status}] {self.total} tests, {self.failures} failures, {self.errors} errors"
+
+
+class TestRunner:
+    """Runs the project test suite via ``pytest``."""
+
+    __test__ = False  # prevent pytest collection
+
+    def __init__(self, project_root: Path, timeout: int = 120) -> None:
+        self._root = project_root.resolve()
+        self._timeout = timeout
+
+    def run(self, extra_args: list[str] | None = None) -> TestResult:
+        """Run pytest and return a :class:`TestResult`.
+
+        Parameters
+        ----------
+        extra_args
+            Additional pytest CLI arguments (e.g. ``["-x", "--tb=short"]``).
+        """
+        cmd = [
+            sys.executable, "-m", "pytest",
+            str(self._root / "semantic_code_intelligence" / "tests"),
+            "-q", "--tb=line", "--no-header",
+        ]
+        if extra_args:
+            cmd.extend(extra_args)
+
+        logger.info("Running: %s", " ".join(cmd))
+        try:
+            proc = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                timeout=self._timeout,
+                cwd=str(self._root),
+            )
+        except subprocess.TimeoutExpired:
+            return TestResult(
+                passed=False,
+                output=f"pytest timed out after {self._timeout}s",
+                return_code=-1,
+            )
+
+        result = TestResult(
+            passed=proc.returncode == 0,
+            output=proc.stdout + proc.stderr,
+            return_code=proc.returncode,
+        )
+
+        # Parse summary line like "2258 passed, 3 warnings in 20.05s"
+        result.total, result.failures, result.errors = _parse_summary(result.output)
+        return result
+
+
+def _parse_summary(output: str) -> tuple[int, int, int]:
+    """Extract passed/failed/error counts from pytest output."""
+    total = 0
+    failures = 0
+    errors = 0
+    for line in reversed(output.splitlines()):
+        line_lower = line.strip().lower()
+        if "passed" in line_lower or "failed" in line_lower or "error" in line_lower:
+            import re
+            m_passed = re.search(r"(\d+)\s+passed", line_lower)
+            m_failed = re.search(r"(\d+)\s+failed", line_lower)
+            m_error = re.search(r"(\d+)\s+error", line_lower)
+            if m_passed:
+                total += int(m_passed.group(1))
+            if m_failed:
+                failures = int(m_failed.group(1))
+                total += failures
+            if m_error:
+                errors = int(m_error.group(1))
+                total += errors
+            break
+    return total, failures, errors

semantic_code_intelligence/indexing/chunker.py

@@ -0,0 +1,174 @@
+"""Code chunker — splits source files into meaningful chunks for embedding."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class CodeChunk:
+    """A chunk of code extracted from a source file."""
+
+    file_path: str
+    content: str
+    start_line: int
+    end_line: int
+    chunk_index: int
+    language: str
+
+
+# Map file extensions to language names
+EXTENSION_TO_LANGUAGE: dict[str, str] = {
+    ".py": "python",
+    ".js": "javascript",
+    ".ts": "typescript",
+    ".jsx": "javascript",
+    ".tsx": "typescript",
+    ".java": "java",
+    ".go": "go",
+    ".rs": "rust",
+    ".c": "c",
+    ".cpp": "cpp",
+    ".h": "c",
+    ".hpp": "cpp",
+    ".rb": "ruby",
+    ".php": "php",
+    ".cs": "csharp",
+    ".swift": "swift",
+    ".kt": "kotlin",
+    ".scala": "scala",
+    ".sh": "shell",
+    ".bash": "shell",
+    ".sql": "sql",
+    ".r": "r",
+    ".lua": "lua",
+    ".dart": "dart",
+    ".ex": "elixir",
+    ".exs": "elixir",
+}
+
+
+def detect_language(file_path: str) -> str:
+    """Detect the programming language from a file extension.
+
+    Args:
+        file_path: Path to the source file.
+
+    Returns:
+        Language name string, or 'unknown' if unrecognized.
+    """
+    ext = Path(file_path).suffix.lower()
+    return EXTENSION_TO_LANGUAGE.get(ext, "unknown")
+
+
+def chunk_code(
+    content: str,
+    file_path: str,
+    chunk_size: int = 512,
+    chunk_overlap: int = 64,
+) -> list[CodeChunk]:
+    """Split source code into overlapping chunks by line boundaries.
+
+    Chunks are split at line boundaries to preserve code structure.
+    Each chunk is at most chunk_size characters, with chunk_overlap
+    characters of overlap with the previous chunk.
+
+    Args:
+        content: The full source code string.
+        file_path: Path to the source file (for metadata).
+        chunk_size: Maximum characters per chunk.
+        chunk_overlap: Characters of overlap between consecutive chunks.
+
+    Returns:
+        List of CodeChunk objects.
+    """
+    if not content.strip():
+        return []
+
+    language = detect_language(file_path)
+    lines = content.splitlines(keepends=True)
+    chunks: list[CodeChunk] = []
+
+    current_chars = 0
+    chunk_start_line = 0
+    chunk_lines: list[str] = []
+    chunk_index = 0
+
+    for i, line in enumerate(lines):
+        chunk_lines.append(line)
+        current_chars += len(line)
+
+        if current_chars >= chunk_size:
+            chunk_text = "".join(chunk_lines)
+            chunks.append(
+                CodeChunk(
+                    file_path=file_path,
+                    content=chunk_text,
+                    start_line=chunk_start_line + 1,  # 1-indexed
+                    end_line=i + 1,
+                    chunk_index=chunk_index,
+                    language=language,
+                )
+            )
+            chunk_index += 1
+
+            # Calculate overlap: walk backwards until we have enough overlap chars
+            overlap_chars = 0
+            overlap_start = len(chunk_lines)
+            for j in range(len(chunk_lines) - 1, -1, -1):
+                overlap_chars += len(chunk_lines[j])
+                if overlap_chars >= chunk_overlap:
+                    overlap_start = j
+                    break
+
+            chunk_lines = chunk_lines[overlap_start:]
+            chunk_start_line = i + 1 - len(chunk_lines) + 1
+            # But we need to preserve 0-indexed line tracking
+            chunk_start_line = (i + 1) - len(chunk_lines)
+            current_chars = sum(len(l) for l in chunk_lines)
+
+    # Emit the last chunk if there's remaining content
+    if chunk_lines:
+        chunk_text = "".join(chunk_lines)
+        if chunk_text.strip():
+            chunks.append(
+                CodeChunk(
+                    file_path=file_path,
+                    content=chunk_text,
+                    start_line=chunk_start_line + 1,
+                    end_line=len(lines),
+                    chunk_index=chunk_index,
+                    language=language,
+                )
+            )
+
+    return chunks
+
+
+def chunk_file(
+    file_path: Path,
+    chunk_size: int = 512,
+    chunk_overlap: int = 64,
+) -> list[CodeChunk]:
+    """Read a file and split it into code chunks.
+
+    Args:
+        file_path: Path to the source file.
+        chunk_size: Maximum characters per chunk.
+        chunk_overlap: Characters of overlap.
+
+    Returns:
+        List of CodeChunk objects.
+    """
+    try:
+        content = file_path.read_text(encoding="utf-8", errors="replace")
+    except (OSError, PermissionError):
+        return []
+
+    return chunk_code(
+        content=content,
+        file_path=str(file_path),
+        chunk_size=chunk_size,
+        chunk_overlap=chunk_overlap,
+    )

semantic_code_intelligence/indexing/parallel.py

@@ -0,0 +1,86 @@
+"""Parallel indexing utilities — concurrent file I/O and chunking.
+
+Speeds up the scanning and chunking phases by processing files in
+parallel using a thread pool, while embedding generation is batched
+through the model (which already uses efficient GPU/CPU batching).
+"""
+
+from __future__ import annotations
+
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+from typing import Any
+
+from semantic_code_intelligence.indexing.chunker import CodeChunk, chunk_file
+from semantic_code_intelligence.indexing.scanner import ScannedFile, compute_file_hash
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("indexing.parallel")
+
+# Sensible default: don't overwhelm disk or CPU
+DEFAULT_WORKERS = 4
+
+
+def parallel_chunk_files(
+    files: list[ScannedFile],
+    chunk_size: int = 512,
+    chunk_overlap: int = 64,
+    max_workers: int = DEFAULT_WORKERS,
+) -> list[tuple[ScannedFile, list[CodeChunk]]]:
+    """Chunk multiple files in parallel using a thread pool.
+
+    Args:
+        files: List of scanned files to chunk.
+        chunk_size: Max characters per chunk.
+        chunk_overlap: Overlap between consecutive chunks.
+        max_workers: Number of threads.
+
+    Returns:
+        List of (ScannedFile, chunks) tuples in original order.
+    """
+    if not files:
+        return []
+
+    results: dict[int, tuple[ScannedFile, list[CodeChunk]]] = {}
+
+    def _chunk_one(idx: int, sf: ScannedFile) -> tuple[int, ScannedFile, list[CodeChunk]]:
+        chunks = chunk_file(sf.path, chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+        return idx, sf, chunks
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = {
+            executor.submit(_chunk_one, i, sf): i
+            for i, sf in enumerate(files)
+        }
+        for future in as_completed(futures):
+            idx, sf, chunks = future.result()
+            results[idx] = (sf, chunks)
+
+    return [results[i] for i in range(len(files))]
+
+
+def parallel_scan_hashes(
+    file_paths: list[Path],
+    max_workers: int = DEFAULT_WORKERS,
+) -> dict[Path, str]:
+    """Compute file hashes in parallel.
+
+    Args:
+        file_paths: Files to hash.
+        max_workers: Number of threads.
+
+    Returns:
+        Mapping of path → SHA-256 hex digest.
+    """
+    result: dict[Path, str] = {}
+
+    def _hash_one(p: Path) -> tuple[Path, str]:
+        return p, compute_file_hash(p)
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(_hash_one, p) for p in file_paths]
+        for future in as_completed(futures):
+            p, h = future.result()
+            result[p] = h
+
+    return result