reducio 1.0.0__py3-none-any.whl

reducio/__init__.py

@@ -0,0 +1,3 @@
+"""reducio — semantic code compression engine."""
+
+__version__ = "1.0.0"

reducio/__main__.py

@@ -0,0 +1,4 @@
+from reducio.cli import main
+
+if __name__ == "__main__":
+    main()

reducio/agents/__init__.py

@@ -0,0 +1,17 @@
+"""
+Agent implementations for code analysis and refactoring.
+"""
+
+from reducio.agents.analyzer import AnalyzerAgent
+from reducio.agents.deduplicator import DeduplicatorAgent
+from reducio.agents.idiomatizer import IdiomatizerAgent
+from reducio.agents.pattern import PatternAgent
+from reducio.agents.quality_checker import QualityCheckerAgent
+
+__all__ = [
+    "AnalyzerAgent",
+    "DeduplicatorAgent",
+    "IdiomatizerAgent",
+    "PatternAgent",
+    "QualityCheckerAgent",
+]

reducio/agents/analyzer.py

@@ -0,0 +1,17 @@
+"""Analyzer agent using the shared, versioned source metrics."""
+
+from reducio.analysis import analyze_files
+from reducio.models import AnalyzeRequest, AnalyzeResult
+from reducio.workspace import Workspace
+
+
+class AnalyzerAgent:
+    def __init__(self, workspace: Workspace):
+        self.workspace = workspace
+
+    async def analyze(self, request: AnalyzeRequest) -> AnalyzeResult:
+        return analyze_files(
+            request.files or self.workspace.list_files(),
+            self.workspace.cfg,
+            request.path,
+        )

reducio/agents/base.py

@@ -0,0 +1,138 @@
+"""Base agent class."""
+
+from __future__ import annotations
+
+import uuid
+from typing import TYPE_CHECKING
+
+from reducio.models import FileChange, PlanDiagnostic, PlanningProvenance, RefactorPlan
+from reducio.plan_review import validate_plan
+from reducio.session import SessionStore
+from reducio.utils.code_utils import strip_code_fence
+from reducio.workspace import Workspace
+
+if TYPE_CHECKING:
+    from reducio.llm.router import LLMRouter
+
+
+class ModelRewriteError(RuntimeError):
+    pass
+
+
+class BaseAgent:
+    def __init__(
+        self,
+        workspace: Workspace | None = None,
+        llm_router: LLMRouter | None = None,
+        session_store: SessionStore | None = None,
+    ):
+        self.workspace = workspace
+        self.llm = llm_router
+        self.session_store = session_store or SessionStore(
+            str(workspace.root / ".reducio" / "sessions") if workspace else ".reducio/sessions"
+        )
+        self._session_plans: dict[str, RefactorPlan] = {}
+        self._begin_plan()
+
+    def _begin_plan(self, allow_fallback: bool = False) -> None:
+        self.allow_fallback = allow_fallback
+        self.diagnostics: list[PlanDiagnostic] = []
+        self.provenance: list[PlanningProvenance] = []
+
+    def _generate_session_id(self) -> str:
+        return str(uuid.uuid4())
+
+    def _save_plan(self, plan: RefactorPlan, command_type: str) -> None:
+        self.session_store.save_plan(plan, command_type=command_type)
+        self._session_plans[plan.session_id] = plan
+
+    def get_plan(self, session_id: str) -> RefactorPlan | None:
+        return self.session_store.load_plan(session_id)
+
+    def _file_content_path(self, file) -> tuple[str, str]:
+        if hasattr(file, "content"):
+            return file.content, file.path
+        return file["content"], file["path"]
+
+    def _llm_enabled(self) -> bool:
+        # An explicit model request must not silently degrade when no router is wired.
+        return bool(self.workspace and self.workspace.cfg.model)
+
+    async def _llm_rewrite(
+        self, content: str, path: str, instruction: str, description: str
+    ) -> FileChange | None:
+        """Ask the LLM to rewrite a whole module; returns a reviewable change or None."""
+        prompt = (
+            f"{instruction}\nReturn ONLY the complete rewritten module, no prose.\n\n"
+            f"```python\n{content}\n```"
+        )
+        model = self.workspace.cfg.model if self.workspace else ""
+        model = "custom endpoint" if "://" in model else model
+        try:
+            if self.llm is None:
+                raise ModelRewriteError("No model router available")
+            raw = await self.llm.complete(
+                prompt, system_prompt="You are an expert Python engineer."
+            )
+            code = strip_code_fence(raw)
+            if not code.strip():
+                raise ValueError("Empty model response")
+            compile(code, path, "exec")
+        except Exception:
+            self.diagnostics.append(
+                PlanDiagnostic(
+                    code="model_failed",
+                    file=path,
+                    message="Model rewrite failed or returned empty/invalid Python"
+                    + (
+                        "; explicit fallback enabled"
+                        if self.allow_fallback
+                        else "; no application permitted"
+                    ),
+                    severity="warning" if self.allow_fallback else "error",
+                )
+            )
+            self.provenance.append(
+                PlanningProvenance(file=path, engine="model", outcome="failed", model=model)
+            )
+            raise ModelRewriteError("Model rewrite unavailable") from None
+        self.provenance.append(
+            PlanningProvenance(
+                file=path,
+                engine="model",
+                outcome="unchanged" if code.strip() == content.strip() else "proposed",
+                model=model,
+            )
+        )
+        if code.strip() == content.strip():
+            return None
+        return FileChange(
+            path=path,
+            original=content,
+            modified=code if code.endswith("\n") else code + "\n",
+            description=description,
+        )
+
+    def _finalize_plan(
+        self, changes: list, description: str, command_type: str, **plan_kw
+    ) -> RefactorPlan:
+        plan = RefactorPlan(
+            session_id=self._generate_session_id(),
+            changes=changes,
+            description=description,
+            diagnostics=self.diagnostics,
+            provenance=self.provenance,
+            **plan_kw,
+        )
+        # Coalesce truly identical changes, but never silently pick one conflicting version.
+        unique = []
+        for change in plan.changes:
+            if change not in unique:
+                unique.append(change)
+        plan.changes = unique
+        plan.diagnostics.extend(
+            validate_plan(plan, self.workspace.root if self.workspace else None)
+        )
+        plan.complete = not any(item.severity == "error" for item in plan.diagnostics)
+        self._save_plan(plan, command_type)
+        return plan

reducio/agents/deduplicator.py

@@ -0,0 +1,194 @@
+"""Deduplicator agent — semantic duplicate detection."""
+
+from __future__ import annotations
+
+import ast
+import builtins
+import symtable
+from typing import TYPE_CHECKING
+
+from reducio.agents.base import BaseAgent
+
+if TYPE_CHECKING:
+    from reducio.embeddings.service import EmbeddingService
+from reducio.models import (
+    CodeBlock,
+    DeduplicateRequest,
+    FileChange,
+    FileInfo,
+    Language,
+    PlanDiagnostic,
+    PlanningProvenance,
+    RefactorPlan,
+)
+from reducio.parse import ParserError, get_complexity
+from reducio.plan_review import advisory_path
+from reducio.repo import detect_language
+from reducio.session import SessionStore
+from reducio.workspace import Workspace
+
+
+class DeduplicatorAgent(BaseAgent):
+    workspace: Workspace
+
+    def __init__(
+        self,
+        workspace: Workspace,
+        embedding_service: EmbeddingService,
+        llm_router=None,
+        session_store: SessionStore | None = None,
+    ):
+        super().__init__(workspace, llm_router, session_store)
+        self.embedding_service = embedding_service
+
+    async def find_duplicates(self, request: DeduplicateRequest) -> RefactorPlan:
+        self._begin_plan()
+        files = request.files or self.workspace.list_files()
+        if not self.embedding_service.is_using_real_embeddings:
+            self.diagnostics.append(
+                PlanDiagnostic(
+                    code="embeddings_unavailable",
+                    severity="error",
+                    message="Semantic embeddings unavailable; install reducio[embeddings] and retry.",
+                )
+            )
+            return self._finalize_plan(
+                [],
+                "Semantic embeddings unavailable; no duplicates analyzed. See installation docs for the embeddings extra.",
+                "deduplicate",
+            )
+        blocks = self._extract_blocks(files)
+        try:
+            groups = await self.embedding_service.find_duplicates(
+                blocks, request.similarity_threshold
+            )
+        except Exception:
+            self.diagnostics.append(
+                PlanDiagnostic(
+                    code="embeddings_failed",
+                    severity="error",
+                    message="Semantic duplicate search failed; no application permitted.",
+                )
+            )
+            groups = []
+        self.provenance.append(
+            PlanningProvenance(
+                file="",
+                engine="embeddings",
+                outcome=(
+                    "failed" if any(d.severity == "error" for d in self.diagnostics) else "scanned"
+                ),
+            )
+        )
+        changes = []
+        for group in groups:
+            if len(group) >= 2:
+                ch = self._create_dedup_change(group)
+                if ch:
+                    changes.append(ch)
+        return self._finalize_plan(
+            changes,
+            f"Found {len(groups)} duplicate group(s); proposing {len(changes)} shared-util "
+            "suggestion(s) (review before adopting — call sites are not rewritten).",
+            "deduplicate",
+        )
+
+    def _extract_blocks(self, files: list[FileInfo]) -> list[CodeBlock]:
+        blocks: list[CodeBlock] = []
+        for f in files:
+            lang = detect_language(f.path)
+            if lang == Language.UNKNOWN:
+                continue
+            try:
+                self.workspace.get_symbols(f.path, f.content)
+                tree = ast.parse(f.content)
+                module_scope = symtable.symtable(f.content, f.path, "exec")
+            except ParserError, SyntaxError, ValueError:
+                self.diagnostics.append(
+                    PlanDiagnostic(
+                        code="parser_failed",
+                        file=f.path,
+                        severity="error",
+                        message="Python parsing failed; duplicate analysis is incomplete.",
+                    )
+                )
+                continue
+            top = [n for n in tree.body if isinstance(n, (ast.FunctionDef, ast.AsyncFunctionDef))]
+            for node in ast.walk(tree):
+                if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) and node not in top:
+                    self.diagnostics.append(
+                        PlanDiagnostic(
+                            code="unsupported_scope",
+                            file=f.path,
+                            message=f"Skipped {node.name}: methods and nested functions are not standalone utilities.",
+                        )
+                    )
+            for node in top:
+                content = ast.get_source_segment(f.content, node) or ""
+                bindings = {
+                    s.get_name()
+                    for s in module_scope.get_symbols()
+                    if s.is_assigned() or s.is_imported()
+                }
+                dependencies = _dependencies(content, node.name, bindings)
+                if node.decorator_list or dependencies:
+                    self.diagnostics.append(
+                        PlanDiagnostic(
+                            code="dependencies",
+                            file=f.path,
+                            message=f"Skipped {node.name}: decorators or external dependencies require review.",
+                        )
+                    )
+                    continue
+                try:
+                    metrics = get_complexity(content)
+                except SyntaxError, ValueError:
+                    continue  # invalid snippets have no trustworthy numeric score
+                blocks.append(
+                    CodeBlock(
+                        id=f"{f.path}:{node.lineno}:{node.name}",
+                        file=f.path,
+                        start_line=node.lineno,
+                        end_line=node.end_lineno or node.lineno,
+                        content=content,
+                        language=lang,
+                        symbol_type="function",
+                        symbol_name=node.name,
+                        metrics=metrics,
+                    )
+                )
+        return blocks
+
+    def _create_dedup_change(self, group: list[CodeBlock]) -> FileChange | None:
+        primary = group[0]
+        return FileChange(
+            path=advisory_path(
+                "utils", primary.file, f"{primary.symbol_name}_{primary.start_line}_dedup"
+            ),
+            original="",
+            modified=primary.content,
+            description=(
+                f"Proposed shared util for '{primary.symbol_name}' from {len(group)} sites "
+                "(suggestion only; applying writes the utility module; "
+                "originals and call sites are not rewritten)"
+            ),
+        )
+
+
+def _dependencies(content: str, name: str, module_bindings: set[str]) -> set[str]:
+    """Conservatively reject names that need the original module's namespace."""
+    table = symtable.symtable(content, "<advisory>", "exec")
+    required: set[str] = set()
+    pending = [table]
+    while pending:
+        scope = pending.pop()
+        required.update(
+            s.get_name()
+            for s in scope.get_symbols()
+            if (s.is_referenced() or s.is_declared_global()) and s.is_global()
+        )
+        pending.extend(scope.get_children())
+    # Recursion is self-contained; builtins need no accompanying import.
+    dynamic = {"globals", "locals", "eval", "exec", "__import__"}
+    allowed_builtins = set(dir(builtins)) - module_bindings - dynamic
+    return required - allowed_builtins - {name}

reducio/agents/idiomatizer.py

@@ -0,0 +1,243 @@
+"""
+Idiomatizer agent for transforming code to idiomatic patterns (Python heuristics).
+"""
+
+import ast
+import re
+
+from reducio.agents.base import BaseAgent, ModelRewriteError
+from reducio.models import (
+    FileChange,
+    IdiomatizeRequest,
+    Language,
+    PlanDiagnostic,
+    PlanningProvenance,
+    RefactorPlan,
+)
+from reducio.repo import detect_language
+from reducio.session import SessionStore
+
+
+class IdiomatizerAgent(BaseAgent):
+    def __init__(self, workspace=None, llm_router=None, session_store: SessionStore | None = None):
+        super().__init__(workspace, llm_router, session_store)
+
+    async def idiomatize(self, request: IdiomatizeRequest) -> RefactorPlan:
+        self._begin_plan(request.allow_fallback)
+        changes = []
+        idioms = 0
+        for file in request.files:
+            try:
+                change, count = await self._idiomatize_file(file)
+            except ModelRewriteError:
+                break
+            if change:
+                changes.append(change)
+                idioms += count
+        return self._finalize_plan(
+            changes,
+            f"Found {idioms} opportunities for idiomatic improvements in {len(changes)} file(s).",
+            "idiomatize",
+        )
+
+    async def _idiomatize_file(self, file) -> tuple[FileChange | None, int]:
+        content, path = self._file_content_path(file)
+        if detect_language(path) != Language.PYTHON:
+            return None, 0
+        try:
+            ast.parse(content)  # can't safely rewrite (or validate) a file that doesn't parse
+        except SyntaxError:
+            self.diagnostics.append(
+                PlanDiagnostic(
+                    code="invalid_source", file=path, message="Skipped invalid Python source"
+                )
+            )
+            return None, 0
+        if self._llm_enabled():
+            try:
+                change = await self._llm_rewrite(
+                    content,
+                    path,
+                    "Rewrite the following Python module to be more idiomatic and concise without changing behaviour.",
+                    "LLM idiomatic rewrite",
+                )
+                return change, int(change is not None)
+            except ModelRewriteError:
+                if not self.allow_fallback:
+                    raise
+        change, count = self._idiomatize_python(content, path)
+        self.provenance.append(
+            PlanningProvenance(
+                file=path, engine="heuristic", outcome="proposed" if change else "unchanged"
+            )
+        )
+        return change, count
+
+    def _idiomatize_python(self, content: str, path: str) -> tuple[FileChange | None, int]:
+        """Collect every idiom as a line span and emit one whole-file change.
+
+        Spans are applied to a copy of the file in reverse so earlier line
+        numbers stay valid, and the resulting diff is file-relative (the old
+        per-snippet changes produced diffs that applied at line 1 — see ROADMAP P0).
+        """
+        lines = content.split("\n")
+        spans = []  # (start, end, replacement, description)
+        i = 0
+        while i < len(lines):
+            result = self._try_python_idiom(lines[i], lines, i)
+            if result:
+                original, modified, description = result
+                consumed = original.count("\n") + 1
+                spans.append((i, i + consumed, modified, f"L{i + 1}: {description}"))
+                i += consumed
+            else:
+                i += 1
+        if not spans:
+            return None, 0
+        new_lines = list(lines)
+        for start, end, replacement, _ in reversed(spans):
+            new_lines[start:end] = replacement.split("\n")
+        return (
+            FileChange(
+                path=path,
+                original=content,
+                modified="\n".join(new_lines),
+                description="; ".join(desc for *_, desc in spans),
+            ),
+            len(spans),
+        )
+
+    def _try_python_idiom(self, line: str, lines: list[str], idx: int) -> tuple | None:
+        return (
+            self._filtered_list_comp(lines, idx)
+            or self._dict_comp(lines, idx)
+            or self._simple_list_comp(lines, idx)
+            or self._compare_to_none(lines, idx)
+            or self._truthiness(lines, idx)
+            or self._or_chain_to_in(lines, idx)
+        )
+
+    @staticmethod
+    def _is_boolean_line(line: str) -> bool:
+        return line.strip().startswith(("if ", "elif ", "while "))
+
+    @staticmethod
+    def _references(var: str, *exprs: str) -> bool:
+        """True if `var` appears as a name in any expr — unsafe to fold into a comprehension.
+
+        A `for/append` loop whose value, iterable, or filter reads the accumulator
+        (e.g. order-preserving dedup: `if v not in u: u.append(v)`) changes behaviour
+        when rewritten as `[v for v in t if v not in u]`, because the comprehension sees
+        an empty `u`. Refuse those.
+        """
+        pat = re.compile(rf"\b{re.escape(var)}\b")
+        return any(pat.search(e) for e in exprs)
+
+    def _truthiness(self, lines: list[str], idx: int) -> tuple | None:
+        line = lines[idx]
+        if not self._is_boolean_line(line):
+            return None
+        new = re.sub(r"len\(([^()]+)\)\s*>\s*0", r"\1", line)
+        new = re.sub(r"len\(([^()]+)\)\s*==\s*0", r"not \1", new)
+        if new == line:
+            return None
+        return line, new, "Use truthiness instead of a len() comparison"
+
+    def _or_chain_to_in(self, lines: list[str], idx: int) -> tuple | None:
+        line = lines[idx]
+        m = re.match(r"(\s*)(if|elif|while)\s+(.+?):\s*$", line)
+        if not m:
+            return None
+        parts = m.group(3).split(" or ")
+        if len(parts) < 2:
+            return None
+        var, values = None, []
+        for part in parts:
+            pm = re.match(r"\s*([\w.]+)\s*==\s*(.+?)\s*$", part)
+            if not pm or (var is not None and pm.group(1) != var):
+                return None
+            var = pm.group(1)
+            values.append(pm.group(2).strip())
+        new = f"{m.group(1)}{m.group(2)} {var} in ({', '.join(values)}):"
+        return line, new, "Use 'in' for repeated equality checks"
+
+    def _simple_list_comp(self, lines: list[str], idx: int) -> tuple | None:
+        if self._is_for_append_pattern(lines, idx):
+            return self._convert_to_list_comp(lines, idx)
+        return None
+
+    def _filtered_list_comp(self, lines: list[str], idx: int) -> tuple | None:
+        if idx + 2 >= len(lines):
+            return None
+        for_m = re.match(r"for\s+(\w+)\s+in\s+(.+?):", lines[idx].strip())
+        if_m = re.match(r"if\s+(.+?):", lines[idx + 1].strip())
+        app_m = re.match(r"(\w+)\.append\((.+)\)", lines[idx + 2].strip())
+        if not (for_m and if_m and app_m):
+            return None
+        if self._references(app_m.group(1), if_m.group(1), app_m.group(2), for_m.group(2)):
+            return None
+        indent = len(lines[idx]) - len(lines[idx].lstrip())
+        comp = (
+            f"{' ' * indent}{app_m.group(1)} = "
+            f"[{app_m.group(2)} for {for_m.group(1)} in {for_m.group(2)} if {if_m.group(1)}]"
+        )
+        return (
+            "\n".join(lines[idx : idx + 3]),
+            comp,
+            "Convert filtered for-loop to list comprehension",
+        )
+
+    def _dict_comp(self, lines: list[str], idx: int) -> tuple | None:
+        if idx + 1 >= len(lines):
+            return None
+        for_m = re.match(r"for\s+(\w+)\s+in\s+(.+?):", lines[idx].strip())
+        assign = re.match(r"(\w+)\[(.+?)\]\s*=\s*(.+)", lines[idx + 1].strip())
+        if not (for_m and assign):
+            return None
+        d = assign.group(1)
+        # Only a dict literal supports comprehension rewrite; lists use index assignment too.
+        if not any(re.match(r"\s*" + re.escape(d) + r"\s*=\s*\{\}\s*$", ln) for ln in lines[:idx]):
+            return None
+        indent = len(lines[idx]) - len(lines[idx].lstrip())
+        comp = (
+            f"{' ' * indent}{d} = "
+            f"{{{assign.group(2)}: {assign.group(3)} for {for_m.group(1)} in {for_m.group(2)}}}"
+        )
+        return f"{lines[idx]}\n{lines[idx + 1]}", comp, "Convert for-loop to dict comprehension"
+
+    def _compare_to_none(self, lines: list[str], idx: int) -> tuple | None:
+        line = lines[idx]
+        new = re.sub(r"==\s*None", "is None", line)
+        new = re.sub(r"!=\s*None", "is not None", new)
+        if new == line:
+            return None
+        return line, new, "Use 'is'/'is not' to compare with None"
+
+    def _is_for_append_pattern(self, lines: list[str], idx: int) -> bool:
+        if idx + 1 >= len(lines):
+            return False
+        line = lines[idx].strip()
+        if not line.startswith("for "):
+            return False
+        return ".append(" in lines[idx + 1].strip()
+
+    def _convert_to_list_comp(self, lines: list[str], idx: int) -> tuple | None:
+        for_line = lines[idx]
+        append_line = lines[idx + 1].strip() if idx + 1 < len(lines) else ""
+        for_match = re.match(r"for\s+(\w+)\s+in\s+(.+?):", for_line.strip())
+        if not for_match:
+            return None
+        append_match = re.match(r"(\w+)\.append\((.+)\)", append_line)
+        if not append_match:
+            return None
+        var, iterable = for_match.group(1), for_match.group(2)
+        list_var, expr = append_match.group(1), append_match.group(2)
+        if self._references(list_var, expr, iterable):
+            return None
+        indent = len(for_line) - len(for_line.lstrip())
+        list_comp = f"{' ' * indent}{list_var} = [{expr} for {var} in {iterable}]"
+        return (
+            f"{for_line}\n{append_line}",
+            list_comp,
+            "Convert for-loop with append to list comprehension",
+        )