codexa 0.4.0__py3-none-any.whl

semantic_code_intelligence/llm/cross_refactor.py

@@ -0,0 +1,272 @@
+"""Cross-repo refactoring suggestions.
+
+Compares symbols across workspace repositories to find:
+- Duplicate or near-duplicate logic that could be shared
+- Inconsistent API patterns across repos
+- Refactoring opportunities informed by cross-repo dependency analysis
+
+Uses the Workspace multi-repo search and per-repo ContextBuilder to
+gather symbols, then asks the LLM for actionable suggestions.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from semantic_code_intelligence.context.engine import ContextBuilder
+from semantic_code_intelligence.llm.provider import LLMMessage, LLMProvider, MessageRole
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("llm.cross_refactor")
+
+
+# ---------------------------------------------------------------------------
+# Data types
+# ---------------------------------------------------------------------------
+
+@dataclass
+class CrossRepoMatch:
+    """A pair of similar symbols found across repos."""
+
+    repo_a: str
+    symbol_a: str
+    file_a: str
+    repo_b: str
+    symbol_b: str
+    file_b: str
+    similarity_note: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "repo_a": self.repo_a,
+            "symbol_a": self.symbol_a,
+            "file_a": self.file_a,
+            "repo_b": self.repo_b,
+            "symbol_b": self.symbol_b,
+            "file_b": self.file_b,
+            "similarity_note": self.similarity_note,
+        }
+
+
+@dataclass
+class CrossRefactorResult:
+    """Result of cross-repo refactoring analysis."""
+
+    repos_analyzed: list[str] = field(default_factory=list)
+    total_symbols: int = 0
+    matches: list[CrossRepoMatch] = field(default_factory=list)
+    suggestions: list[dict[str, Any]] = field(default_factory=list)
+    llm_used: bool = False
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "repos_analyzed": self.repos_analyzed,
+            "total_symbols": self.total_symbols,
+            "matches": [m.to_dict() for m in self.matches],
+            "suggestions": self.suggestions,
+            "llm_used": self.llm_used,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Cross-repo symbol collection
+# ---------------------------------------------------------------------------
+
+def _collect_repo_symbols(
+    repo_name: str,
+    repo_path: Path,
+) -> list[dict[str, Any]]:
+    """Index a single repo and return a flat list of symbol dicts."""
+    builder = ContextBuilder()
+    from semantic_code_intelligence.config.settings import load_config
+    from semantic_code_intelligence.indexing.scanner import scan_repository
+
+    config = load_config(repo_path)
+    for sf in scan_repository(repo_path, config.index):
+        try:
+            builder.index_file(str(repo_path / sf.relative_path))
+        except Exception:
+            logger.debug("Skip unindexable file: %s", sf.relative_path)
+
+    symbols = builder.get_all_symbols()
+    result: list[dict[str, Any]] = []
+    for s in symbols:
+        if s.kind in ("function", "method", "class"):
+            result.append({
+                "repo": repo_name,
+                "name": s.name,
+                "kind": s.kind,
+                "file": s.file_path,
+                "lines": s.end_line - s.start_line + 1,
+                "body": s.body[:600] if s.body else "",
+            })
+    return result
+
+
+# ---------------------------------------------------------------------------
+# Duplicate detection across repos
+# ---------------------------------------------------------------------------
+
+def _find_cross_duplicates(
+    repo_symbols: dict[str, list[dict[str, Any]]],
+    threshold: float = 0.70,
+    min_lines: int = 4,
+) -> list[CrossRepoMatch]:
+    """Find near-duplicate symbols across different repos via trigram Jaccard."""
+    from semantic_code_intelligence.ci.quality import _normalize_body, _trigram_set, _jaccard
+
+    # Build (repo, sym_dict, trigrams) for each candidate
+    candidates: list[tuple[str, dict[str, Any], set[str]]] = []
+    for repo_name, syms in repo_symbols.items():
+        for s in syms:
+            if s["lines"] < min_lines or not s["body"].strip():
+                continue
+            norm = _normalize_body(s["body"])
+            tris = _trigram_set(norm)
+            if tris:
+                candidates.append((repo_name, s, tris))
+
+    matches: list[CrossRepoMatch] = []
+    seen: set[tuple[str, str]] = set()
+
+    for i, (repo_a, sym_a, tris_a) in enumerate(candidates):
+        for j in range(i + 1, len(candidates)):
+            repo_b, sym_b, tris_b = candidates[j]
+            if repo_a == repo_b:
+                continue  # Only cross-repo
+            sim = _jaccard(tris_a, tris_b)
+            if sim >= threshold:
+                _sorted = sorted([f"{repo_a}:{sym_a['name']}", f"{repo_b}:{sym_b['name']}"])
+                pair_key = (_sorted[0], _sorted[1])
+                if pair_key in seen:
+                    continue
+                seen.add(pair_key)
+                matches.append(CrossRepoMatch(
+                    repo_a=repo_a,
+                    symbol_a=sym_a["name"],
+                    file_a=sym_a["file"],
+                    repo_b=repo_b,
+                    symbol_b=sym_b["name"],
+                    file_b=sym_b["file"],
+                    similarity_note=f"Jaccard similarity: {sim:.2f}",
+                ))
+
+    matches.sort(key=lambda m: m.similarity_note, reverse=True)
+    return matches
+
+
+# ---------------------------------------------------------------------------
+# LLM-powered suggestion generation
+# ---------------------------------------------------------------------------
+
+def _generate_suggestions(
+    matches: list[CrossRepoMatch],
+    repo_symbols: dict[str, list[dict[str, Any]]],
+    provider: LLMProvider,
+) -> list[dict[str, Any]]:
+    """Ask the LLM for refactoring suggestions based on cross-repo matches."""
+    if not matches:
+        return []
+
+    # Build a concise summary for the LLM
+    match_text = ""
+    for m in matches[:10]:
+        match_text += (
+            f"- {m.repo_a}/{m.symbol_a} ({m.file_a}) ↔ "
+            f"{m.repo_b}/{m.symbol_b} ({m.file_b}) — {m.similarity_note}\n"
+        )
+
+    repo_summary = ""
+    for repo, syms in repo_symbols.items():
+        repo_summary += f"- {repo}: {len(syms)} symbols\n"
+
+    system = (
+        "You are CodexA, a cross-repository refactoring advisor. Given a list of "
+        "similar symbols found across different repositories, suggest refactoring "
+        "opportunities. Return a JSON list of objects with keys: 'title', "
+        "'description', 'affected_repos', 'priority' (high/medium/low)."
+    )
+    user_msg = (
+        f"Repositories:\n{repo_summary}\n"
+        f"Similar symbols across repos:\n{match_text}\n"
+        "Suggest refactoring strategies (e.g., extract shared lib, unify APIs)."
+    )
+
+    import json
+
+    messages = [
+        LLMMessage(role=MessageRole.SYSTEM, content=system),
+        LLMMessage(role=MessageRole.USER, content=user_msg),
+    ]
+    resp = provider.chat(messages)
+
+    try:
+        parsed = json.loads(resp.content)
+        if isinstance(parsed, list):
+            result: list[dict[str, Any]] = parsed
+            return result
+        if isinstance(parsed, dict) and "suggestions" in parsed:
+            suggestions: list[dict[str, Any]] = parsed["suggestions"]
+            return suggestions
+    except (json.JSONDecodeError, TypeError):
+        pass
+
+    return [{"title": "See raw analysis", "description": resp.content, "priority": "medium"}]
+
+
+# ---------------------------------------------------------------------------
+# Public API
+# ---------------------------------------------------------------------------
+
+def analyze_cross_repo(
+    workspace_root: Path,
+    *,
+    provider: LLMProvider | None = None,
+    threshold: float = 0.70,
+    repos: list[str] | None = None,
+) -> CrossRefactorResult:
+    """Analyse a workspace for cross-repo refactoring opportunities.
+
+    Args:
+        workspace_root: Workspace root containing ``.codexa/workspace.json``.
+        provider: Optional LLM provider for generating suggestions.
+        threshold: Jaccard similarity threshold for duplicate detection.
+        repos: Restrict to these repo names. None = all registered repos.
+
+    Returns:
+        CrossRefactorResult with matches and optional LLM suggestions.
+    """
+    from semantic_code_intelligence.workspace import Workspace
+
+    try:
+        ws = Workspace.load(workspace_root)
+    except FileNotFoundError:
+        return CrossRefactorResult()
+
+    targets = repos or [r.name for r in ws.repos]
+    repo_symbols: dict[str, list[dict[str, Any]]] = {}
+
+    for rname in targets:
+        entry = ws.get_repo(rname)
+        if entry is None:
+            continue
+        repo_symbols[rname] = _collect_repo_symbols(rname, Path(entry.path))
+
+    total_symbols = sum(len(s) for s in repo_symbols.values())
+    matches = _find_cross_duplicates(repo_symbols, threshold=threshold)
+
+    suggestions: list[dict[str, Any]] = []
+    llm_used = False
+    if provider and matches:
+        suggestions = _generate_suggestions(matches, repo_symbols, provider)
+        llm_used = True
+
+    return CrossRefactorResult(
+        repos_analyzed=targets,
+        total_symbols=total_symbols,
+        matches=matches,
+        suggestions=suggestions,
+        llm_used=llm_used,
+    )

semantic_code_intelligence/llm/investigation.py

@@ -0,0 +1,274 @@
+"""Autonomous investigation chains — multi-step code exploration.
+
+An **InvestigationChain** drives an iterative loop:
+
+1. Formulate a search query from the user's question.
+2. Gather context (semantic search, symbol lookup, dependency analysis).
+3. Ask the LLM to evaluate what was found and decide the next action.
+4. Repeat until the LLM signals "conclude" or a step limit is reached.
+
+Each step is recorded as a ``ReasoningStep`` in ``SessionMemory`` so
+the chain is fully transparent and reproducible.
+"""
+
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Any
+
+from semantic_code_intelligence.context.engine import ContextBuilder
+from semantic_code_intelligence.context.memory import ReasoningStep, SessionMemory
+from semantic_code_intelligence.llm.provider import LLMMessage, LLMProvider, MessageRole
+from semantic_code_intelligence.services.search_service import search_codebase
+from semantic_code_intelligence.utils.logging import get_logger
+
+logger = get_logger("llm.investigation")
+
+
+# ---------------------------------------------------------------------------
+# Data types
+# ---------------------------------------------------------------------------
+
+@dataclass
+class InvestigationResult:
+    """Final result of an investigation chain."""
+
+    question: str
+    conclusion: str
+    steps: list[dict[str, Any]] = field(default_factory=list)
+    chain_id: str = ""
+    total_steps: int = 0
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "question": self.question,
+            "conclusion": self.conclusion,
+            "steps": self.steps,
+            "chain_id": self.chain_id,
+            "total_steps": self.total_steps,
+        }
+
+
+# ---------------------------------------------------------------------------
+# Action helpers
+# ---------------------------------------------------------------------------
+
+def _action_search(query: str, project_root: Path, top_k: int = 5) -> str:
+    """Run semantic search and return a text summary."""
+    try:
+        results = search_codebase(query, project_root, top_k=top_k, threshold=0.2)
+        if not results:
+            return "No results found."
+        parts: list[str] = []
+        for r in results:
+            d = r.to_dict()
+            parts.append(
+                f"[{d.get('file_path', '?')}] (score {d.get('score', 0):.2f})\n"
+                f"{d.get('content', d.get('chunk', ''))[:500]}"
+            )
+        return "\n---\n".join(parts)
+    except Exception:
+        return "Semantic search unavailable."
+
+
+def _action_analyze(symbol_name: str, builder: ContextBuilder) -> str:
+    """Look up a symbol and return its context."""
+    matches = builder.find_symbol(symbol_name)
+    if not matches:
+        return f"Symbol '{symbol_name}' not found."
+    parts: list[str] = []
+    for s in matches[:3]:
+        parts.append(
+            f"{s.kind} {s.name} in {s.file_path} (L{s.start_line}-{s.end_line})\n"
+            f"{s.body[:400]}"
+        )
+    return "\n---\n".join(parts)
+
+
+def _action_deps(file_path: str, builder: ContextBuilder, project_root: Path) -> str:
+    """Gather dependency info for a file."""
+    from semantic_code_intelligence.context.engine import DependencyMap
+
+    dm = DependencyMap()
+    full = project_root / file_path if not Path(file_path).is_absolute() else Path(file_path)
+    if full.exists():
+        deps = dm.add_file(str(full))
+        parts = [f"{d.import_text} (L{d.line})" for d in deps]
+        return "Dependencies: " + ", ".join(parts) if parts else "No dependencies found."
+    return f"File not found: {file_path}"
+
+
+# ---------------------------------------------------------------------------
+# Investigation chain
+# ---------------------------------------------------------------------------
+
+_PLANNER_SYSTEM = """\
+You are CodexA, an autonomous code investigation agent. Your task is to
+answer the user's question by systematically exploring the codebase.
+
+On each turn you will receive context gathered from the previous action.
+Respond with a JSON object with exactly these keys:
+- "thought": one sentence about what you learned and what to do next
+- "action": one of "search", "analyze", "deps", "conclude"
+- "action_input": the argument for the action (search query, symbol name, file path, or final answer)
+
+When you have enough information, use action "conclude" and put your
+final answer in "action_input".
+"""
+
+
+class InvestigationChain:
+    """Drives an iterative search-analyze-conclude loop via LLM."""
+
+    def __init__(
+        self,
+        provider: LLMProvider,
+        project_root: Path,
+        *,
+        builder: ContextBuilder | None = None,
+        memory: SessionMemory | None = None,
+        max_steps: int = 6,
+    ) -> None:
+        self._provider = provider
+        self._root = project_root.resolve()
+        self._builder = builder or ContextBuilder()
+        self._memory = memory or SessionMemory()
+        self._max_steps = max_steps
+        self._indexed = False
+
+    def _ensure_indexed(self) -> None:
+        if self._indexed:
+            return
+        from semantic_code_intelligence.config.settings import load_config
+        from semantic_code_intelligence.indexing.scanner import scan_repository
+
+        config = load_config(self._root)
+        for sf in scan_repository(self._root, config.index):
+            try:
+                self._builder.index_file(str(self._root / sf.relative_path))
+            except Exception:
+                logger.debug("Skip unindexable file: %s", sf.relative_path)
+        self._indexed = True
+
+    def _run_action(self, action: str, action_input: str) -> str:
+        """Execute an action and return its text output."""
+        if action == "search":
+            return _action_search(action_input, self._root)
+        elif action == "analyze":
+            self._ensure_indexed()
+            return _action_analyze(action_input, self._builder)
+        elif action == "deps":
+            self._ensure_indexed()
+            return _action_deps(action_input, self._builder, self._root)
+        return action_input
+
+    def _parse_plan(self, text: str) -> dict[str, str]:
+        """Parse the LLM planner response into {thought, action, action_input}."""
+        # Try JSON first
+        try:
+            parsed = json.loads(text)
+            if isinstance(parsed, dict) and "action" in parsed:
+                return {
+                    "thought": str(parsed.get("thought", "")),
+                    "action": str(parsed.get("action", "conclude")),
+                    "action_input": str(parsed.get("action_input", text)),
+                }
+        except (json.JSONDecodeError, TypeError):
+            pass
+        # Fallback — treat entire response as conclusion
+        return {"thought": "", "action": "conclude", "action_input": text}
+
+    def _conclude_streaming(self, messages: list[LLMMessage]) -> str:
+        """Stream the final conclusion tokens to stdout, return accumulated text."""
+        from semantic_code_intelligence.llm.streaming import stream_chat
+
+        gen = stream_chat(self._provider, messages)
+        accumulated = ""
+        import sys
+        for event in gen:
+            if event.kind == "token":
+                accumulated += event.content
+                sys.stdout.write(event.content)
+                sys.stdout.flush()
+        sys.stdout.write("\n")
+        return accumulated
+
+    def investigate(self, question: str, *, stream_conclusion: bool = False) -> InvestigationResult:
+        """Run a full investigation loop and return the result.
+
+        Args:
+            question: The question to investigate.
+            stream_conclusion: If True, yield the conclusion token-by-token
+                via ``stream_chat`` and print incrementally.
+        """
+        chain_id = uuid.uuid4().hex[:10]
+        self._memory.start_chain(chain_id)
+
+        messages: list[LLMMessage] = [
+            LLMMessage(role=MessageRole.SYSTEM, content=_PLANNER_SYSTEM),
+            LLMMessage(role=MessageRole.USER, content=f"Question: {question}"),
+        ]
+
+        steps: list[dict[str, Any]] = []
+        conclusion = ""
+
+        for step_num in range(1, self._max_steps + 1):
+            # Ask the planner
+            resp = self._provider.chat(messages)
+            plan = self._parse_plan(resp.content)
+
+            action = plan["action"]
+            action_input = plan["action_input"]
+            thought = plan["thought"]
+
+            if action == "conclude":
+                conclusion = action_input
+                self._memory.add_step(chain_id, "conclude", question, conclusion)
+                steps.append({
+                    "step": step_num,
+                    "action": "conclude",
+                    "thought": thought,
+                    "output": conclusion,
+                })
+                break
+
+            # Execute the action
+            output = self._run_action(action, action_input)
+            self._memory.add_step(chain_id, action, action_input, output)
+            steps.append({
+                "step": step_num,
+                "action": action,
+                "action_input": action_input,
+                "thought": thought,
+                "output": output[:500],
+            })
+
+            # Feed result back to planner
+            messages.append(LLMMessage(role=MessageRole.ASSISTANT, content=resp.content))
+            messages.append(LLMMessage(
+                role=MessageRole.USER,
+                content=f"Action result:\n{output[:2000]}",
+            ))
+        else:
+            # Exhausted steps — ask for final conclusion
+            messages.append(LLMMessage(
+                role=MessageRole.USER,
+                content="You have reached the step limit. Please provide your best conclusion now.",
+            ))
+            if stream_conclusion:
+                conclusion = self._conclude_streaming(messages)
+            else:
+                resp = self._provider.chat(messages)
+            conclusion = resp.content
+            self._memory.add_step(chain_id, "conclude", "forced", conclusion)
+
+        return InvestigationResult(
+            question=question,
+            conclusion=conclusion,
+            steps=steps,
+            chain_id=chain_id,
+            total_steps=len(steps),
+        )

semantic_code_intelligence/llm/mock_provider.py

@@ -0,0 +1,77 @@
+"""Mock LLM provider — deterministic responses for testing."""
+
+from __future__ import annotations
+
+from typing import Any
+
+from semantic_code_intelligence.llm.provider import (
+    LLMMessage,
+    LLMProvider,
+    LLMResponse,
+)
+
+
+class MockProvider(LLMProvider):
+    """A mock LLM provider that returns configurable responses.
+
+    Useful for unit tests and offline development without a live LLM.
+    """
+
+    def __init__(
+        self,
+        default_response: str = "This is a mock LLM response.",
+        model: str = "mock-model",
+    ) -> None:
+        self._default_response = default_response
+        self._model = model
+        self._call_history: list[dict[str, Any]] = []
+        self._response_queue: list[str] = []
+
+    @property
+    def name(self) -> str:
+        return "mock"
+
+    @property
+    def call_history(self) -> list[dict[str, Any]]:
+        """Return a log of all calls made to this mock."""
+        return list(self._call_history)
+
+    def enqueue_response(self, response: str) -> None:
+        """Enqueue a custom response.  FIFO — next call pops from front."""
+        self._response_queue.append(response)
+
+    def _next_response(self) -> str:
+        if self._response_queue:
+            return self._response_queue.pop(0)
+        return self._default_response
+
+    def complete(self, prompt: str, **kwargs: Any) -> LLMResponse:
+        content = self._next_response()
+        self._call_history.append({
+            "method": "complete",
+            "prompt": prompt,
+            "kwargs": kwargs,
+            "response": content,
+        })
+        return LLMResponse(
+            content=content,
+            model=self._model,
+            provider=self.name,
+            usage={"prompt_tokens": len(prompt) // 4, "completion_tokens": len(content) // 4, "total_tokens": (len(prompt) + len(content)) // 4},
+        )
+
+    def chat(self, messages: list[LLMMessage], **kwargs: Any) -> LLMResponse:
+        content = self._next_response()
+        self._call_history.append({
+            "method": "chat",
+            "messages": [m.to_dict() for m in messages],
+            "kwargs": kwargs,
+            "response": content,
+        })
+        total_chars = sum(len(m.content) for m in messages) + len(content)
+        return LLMResponse(
+            content=content,
+            model=self._model,
+            provider=self.name,
+            usage={"prompt_tokens": total_chars // 4, "completion_tokens": len(content) // 4, "total_tokens": total_chars // 4},
+        )