devsper 2.1.6__py3-none-any.whl

devsper/dev/builder.py

@@ -0,0 +1,195 @@
+"""
+Autonomous Application Builder: build a working repo from an app description.
+
+Flow:
+1) architecture design
+2) repo scaffold
+3) module implementation
+4) test generation
+5) test execution
+6) debugging loop
+"""
+
+from pathlib import Path
+
+from devsper.dev.scaffold import ArchitecturePlan, scaffold_repo
+from devsper.dev.sandbox import Sandbox, SandboxLimits
+from devsper.dev.debugger import debug_loop
+from devsper.dev.repo_index import RepoIndex
+
+
+def _design_architecture(description: str) -> ArchitecturePlan:
+    """Produce an architecture plan from app description (heuristic + optional LLM)."""
+    desc_lower = (description or "").lower()
+    name = (description or "app").strip()[:50].replace("/", "-")
+    backend = "fastapi"
+    frontend = "none"
+    if "flask" in desc_lower:
+        backend = "flask"
+    if "react" in desc_lower or "frontend" in desc_lower or "ui" in desc_lower:
+        frontend = "react"
+    if "todo" in desc_lower:
+        name = "Todo App"
+    return ArchitecturePlan(
+        name=name or "App",
+        description=description or "Generated application",
+        backend=backend,
+        frontend=frontend,
+        features=[],
+    )
+
+
+def _implement_modules(sandbox: Sandbox, plan: ArchitecturePlan, description: str) -> None:
+    """Generate and write main backend module(s)."""
+    if "fastapi" in plan.backend.lower() and "todo" in plan.name.lower():
+        main_py = '''"""FastAPI Todo API."""
+from fastapi import FastAPI, HTTPException
+from pydantic import BaseModel
+from typing import List
+
+app = FastAPI(title="Todo API")
+
+class TodoItem(BaseModel):
+    id: int
+    title: str
+    done: bool = False
+
+todos: List[TodoItem] = []
+next_id = 1
+
+@app.get("/")
+def root():
+    return {"message": "Todo API", "docs": "/docs"}
+
+@app.get("/todos")
+def list_todos():
+    return {"todos": [t.model_dump() for t in todos]}
+
+@app.post("/todos")
+def create_todo(title: str):
+    global next_id
+    item = TodoItem(id=next_id, title=title, done=False)
+    next_id += 1
+    todos.append(item)
+    return item.model_dump()
+
+@app.patch("/todos/{item_id}")
+def toggle_todo(item_id: int):
+    for t in todos:
+        if t.id == item_id:
+            t.done = not t.done
+            return t.model_dump()
+    raise HTTPException(status_code=404, detail="Not found")
+
+@app.delete("/todos/{item_id}")
+def delete_todo(item_id: int):
+    global todos
+    for i, t in enumerate(todos):
+        if t.id == item_id:
+            todos.pop(i)
+            return {"ok": True}
+    raise HTTPException(status_code=404, detail="Not found")
+'''
+        sandbox.write_file("backend/main.py", main_py)
+    elif "fastapi" in plan.backend.lower():
+        # Generic FastAPI placeholder already from scaffold; leave or minimal enhance
+        pass
+
+
+def _generate_tests(sandbox: Sandbox, plan: ArchitecturePlan) -> None:
+    """Write basic tests."""
+    if "todo" in (plan.name or "").lower():
+        test_py = '''"""Tests for Todo API."""
+import pytest
+from fastapi.testclient import TestClient
+
+# Import app from backend
+import sys
+from pathlib import Path
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent / "backend"))
+from main import app
+
+client = TestClient(app)
+
+def test_root():
+    r = client.get("/")
+    assert r.status_code == 200
+    assert "message" in r.json()
+
+def test_list_todos_empty():
+    r = client.get("/todos")
+    assert r.status_code == 200
+    assert r.json()["todos"] == []
+
+def test_create_and_list_todo():
+    r = client.post("/todos?title=test task")
+    assert r.status_code == 200
+    data = r.json()
+    assert data["title"] == "test task"
+    assert data["done"] is False
+    r2 = client.get("/todos")
+    assert len(r2.json()["todos"]) >= 1
+'''
+        sandbox.write_file("tests/test_app.py", test_py)
+
+
+def run_build(
+    app_description: str,
+    output_dir: str | Path,
+    *,
+    timeout_seconds: int = 300,
+    max_debug_iterations: int = 3,
+) -> dict:
+    """
+    Build a working repository from an app description.
+
+    Steps: architecture design, scaffold, implement, generate tests,
+    install deps, run tests, debug loop until pass or limit.
+
+    Returns dict with keys: success, plan, created_paths, test_passed, debug_result, repo_path.
+    """
+    output_dir = Path(output_dir).resolve()
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    plan = _design_architecture(app_description)
+    created_paths = scaffold_repo(output_dir, plan)
+
+    limits = SandboxLimits(timeout_seconds=timeout_seconds)
+    sandbox = Sandbox(output_dir, limits=limits)
+
+    _implement_modules(sandbox, plan, app_description)
+    _generate_tests(sandbox, plan)
+
+    install_result = sandbox.install_dependencies(backend=True, frontend=False)
+
+    def get_fix(stdout: str, stderr: str) -> str:
+        try:
+            from devsper.utils.models import generate
+            from devsper.config import get_config
+            cfg = get_config()
+            model = getattr(cfg.swarm, "worker_model", "gpt-4o") or "gpt-4o"
+            prompt = (
+                "The following test run failed. Suggest a single file fix as:\n"
+                "FILE: <relative path>\nCONTENT: <full file content>\n\n"
+                "Stdout:\n" + (stdout or "")[:2000] + "\n\nStderr:\n" + (stderr or "")[:1000]
+            )
+            return generate(model, prompt)
+        except Exception:
+            return ""
+
+    debug_result = debug_loop(
+        sandbox,
+        max_iterations=max_debug_iterations,
+        test_path="tests",
+        get_fix=get_fix,
+    )
+
+    return {
+        "success": debug_result.passed,
+        "plan": plan,
+        "created_paths": created_paths,
+        "install_success": install_result.success,
+        "test_passed": debug_result.passed,
+        "debug_result": debug_result,
+        "repo_path": str(output_dir),
+    }

devsper/dev/debugger.py

@@ -0,0 +1,95 @@
+"""
+Test execution loop: run tests, detect errors, spawn fix tasks, patch code.
+Loop until tests pass or max iterations.
+"""
+
+from dataclasses import dataclass, field
+from typing import Callable
+
+from devsper.dev.sandbox import Sandbox, SandboxResult
+
+
+@dataclass
+class DebugResult:
+    """Result of the debug loop."""
+
+    passed: bool
+    iterations: int
+    last_stdout: str = ""
+    last_stderr: str = ""
+    fixes_applied: list[str] = field(default_factory=list)
+
+
+def debug_loop(
+    sandbox: Sandbox,
+    max_iterations: int = 5,
+    test_path: str = "tests",
+    get_fix: Callable[[str, str], str] | None = None,
+) -> DebugResult:
+    """
+    Run tests in sandbox; on failure, call get_fix(stdout, stderr) to get patch/code,
+    apply it (e.g. via sandbox.write_file), then re-run. Loop until tests pass or
+    max_iterations. If get_fix is None, no fixes are applied (just re-run).
+    """
+    fixes: list[str] = []
+    last_stdout = ""
+    last_stderr = ""
+
+    for iteration in range(max_iterations):
+        result: SandboxResult = sandbox.run_tests(path=test_path)
+        last_stdout = result.stdout
+        last_stderr = result.stderr or ""
+
+        if result.success:
+            return DebugResult(
+                passed=True,
+                iterations=iteration + 1,
+                last_stdout=last_stdout,
+                last_stderr=last_stderr,
+                fixes_applied=fixes,
+            )
+
+        if get_fix and not result.timed_out:
+            fix_instruction = get_fix(last_stdout, last_stderr)
+            if fix_instruction and fix_instruction.strip():
+                fixes.append(fix_instruction[:500])
+                # get_fix may return "path: <path>\\ncontent: <content>" or similar
+                # A real implementation would parse and call sandbox.write_file;
+                # for now we only record that a fix was requested.
+                # Caller can pass a get_fix that actually writes files.
+                if _apply_fix_instruction(sandbox, fix_instruction):
+                    continue
+        break
+
+    return DebugResult(
+        passed=False,
+        iterations=max_iterations,
+        last_stdout=last_stdout,
+        last_stderr=last_stderr,
+        fixes_applied=fixes,
+    )
+
+
+def _apply_fix_instruction(sandbox: Sandbox, instruction: str) -> bool:
+    """
+    Try to parse a fix instruction (e.g. "FILE: path\\nCONTENT: ...") and apply via sandbox.
+    Returns True if something was applied.
+    """
+    lines = instruction.strip().split("\n")
+    path = None
+    content_lines: list[str] = []
+    in_content = False
+    for line in lines:
+        if line.upper().startswith("FILE:") or line.upper().startswith("PATH:"):
+            path = line.split(":", 1)[1].strip()
+            in_content = False
+        elif line.upper().startswith("CONTENT:") or line.upper().startswith("CODE:"):
+            in_content = True
+            content_lines.append(line.split(":", 1)[1] if ":" in line else "")
+        elif in_content and path:
+            content_lines.append(line)
+    if path and content_lines:
+        content = "\n".join(content_lines)
+        r = sandbox.write_file(path, content)
+        return r.success
+    return False

devsper/dev/repo_index.py

@@ -0,0 +1,138 @@
+"""
+Codebase index for dev agents: AST parsing, dependency graph, symbol search.
+
+Reuses patterns from devsper.tools.code_intelligence; provides a single API
+for agents to understand the generated codebase.
+"""
+
+import ast
+from pathlib import Path
+from dataclasses import dataclass, field
+
+
+@dataclass
+class SymbolInfo:
+    """A top-level symbol (function or class) in a file."""
+
+    name: str
+    kind: str  # "function" or "class"
+    file: str
+    line: int
+
+
+@dataclass
+class DependencyEdge:
+    """Module A depends on module B (import)."""
+
+    from_module: str
+    to_module: str
+
+
+@dataclass
+class RepoIndex:
+    """
+    Index of a repo: files, symbols, dependencies.
+    Built via AST parsing; supports dependency graph and symbol search.
+    """
+
+    root: Path
+    files: list[str] = field(default_factory=list)
+    symbols: list[SymbolInfo] = field(default_factory=list)
+    edges: list[DependencyEdge] = field(default_factory=list)
+    docstrings: dict[str, str] = field(default_factory=dict)
+
+    def _rel_path(self, p: Path) -> str:
+        return str(p.relative_to(self.root)).replace("\\", "/")
+
+    def _imports(self, p: Path) -> list[str]:
+        out: list[str] = []
+        try:
+            tree = ast.parse(p.read_text(encoding="utf-8", errors="replace"))
+            for node in ast.walk(tree):
+                if isinstance(node, ast.Import):
+                    for alias in node.names:
+                        out.append(alias.name.split(".")[0])
+                elif isinstance(node, ast.ImportFrom):
+                    if node.module:
+                        out.append(node.module.split(".")[0])
+        except (SyntaxError, OSError):
+            pass
+        return list(dict.fromkeys(out))
+
+    def _extract_symbols(self, p: Path) -> list[SymbolInfo]:
+        syms: list[SymbolInfo] = []
+        try:
+            tree = ast.parse(p.read_text(encoding="utf-8", errors="replace"))
+            doc = ast.get_docstring(tree)
+            if doc:
+                self.docstrings[self._rel_path(p)] = doc[:500]
+            for node in ast.walk(tree):
+                if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)):
+                    if node.col_offset == 0:
+                        kind = "class" if isinstance(node, ast.ClassDef) else "function"
+                        syms.append(
+                            SymbolInfo(
+                                name=node.name,
+                                kind=kind,
+                                file=self._rel_path(p),
+                                line=node.lineno or 0,
+                            )
+                        )
+        except (SyntaxError, OSError):
+            pass
+        return syms
+
+    def build(self, max_files: int = 200) -> None:
+        """Build index from repo root: parse AST, collect symbols and dependencies."""
+        self.files = []
+        self.symbols = []
+        self.edges = []
+        self.docstrings = {}
+        count = 0
+        for p in self.root.rglob("*.py"):
+            if count >= max_files:
+                break
+            if not p.is_file() or "__pycache__" in p.parts:
+                continue
+            count += 1
+            rel = self._rel_path(p)
+            self.files.append(rel)
+            mod = rel.replace(".py", "").replace("/", ".")
+            for imp in self._imports(p):
+                self.edges.append(DependencyEdge(from_module=mod, to_module=imp))
+            self.symbols.extend(self._extract_symbols(p))
+
+    def dependency_graph(self) -> dict[str, list[str]]:
+        """Return adjacency list: module -> list of imported modules."""
+        out: dict[str, list[str]] = {}
+        for e in self.edges:
+            out.setdefault(e.from_module, []).append(e.to_module)
+        return out
+
+    def symbol_search(self, name: str) -> list[SymbolInfo]:
+        """Find symbols whose name contains the given string."""
+        name_lower = name.lower()
+        return [s for s in self.symbols if name_lower in s.name.lower()]
+
+    def file_search(self, substring: str) -> list[str]:
+        """Find files whose path contains the given string."""
+        sub = substring.lower()
+        return [f for f in self.files if sub in f.lower()]
+
+    def summary(self) -> str:
+        """Human-readable summary for agents."""
+        lines = [
+            f"Root: {self.root}",
+            f"Files: {len(self.files)}",
+            f"Symbols: {len(self.symbols)}",
+            f"Dependencies: {len(self.edges)} edges",
+            "",
+            "Top-level files:",
+        ]
+        for f in sorted(self.files)[:30]:
+            lines.append(f"  {f}")
+        lines.append("")
+        lines.append("Symbols (sample):")
+        for s in self.symbols[:40]:
+            lines.append(f"  {s.kind} {s.name} @ {s.file}:{s.line}")
+        return "\n".join(lines)

devsper/dev/sandbox.py

@@ -0,0 +1,203 @@
+"""
+Code sandbox: write files, run shell commands, run tests, install dependencies.
+
+Enforces timeout and resource limits for safe autonomous execution.
+"""
+
+import os
+import subprocess
+import sys
+import shlex
+from pathlib import Path
+try:
+    import resource
+except ImportError:
+    resource = None  # Windows has no resource module
+from dataclasses import dataclass, field
+
+
+@dataclass
+class SandboxResult:
+    """Result of a sandboxed command or operation."""
+
+    success: bool
+    stdout: str = ""
+    stderr: str = ""
+    returncode: int = 0
+    timed_out: bool = False
+    error: str | None = None
+
+
+@dataclass
+class SandboxLimits:
+    """Resource limits for sandbox execution."""
+
+    timeout_seconds: int = 300
+    max_memory_mb: int | None = 100
+    max_cpu_seconds: int | None = 60  # soft CPU time
+
+
+class Sandbox:
+    """
+    Isolated execution environment with timeout and resource limits.
+
+    Capabilities: write files, run shell commands, run tests, install dependencies.
+    """
+
+    def __init__(self, root: str | Path, limits: SandboxLimits | None = None):
+        self.root = Path(root).resolve()
+        self.limits = limits or SandboxLimits()
+
+    def _cwd(self) -> Path:
+        if not self.root.exists():
+            self.root.mkdir(parents=True, exist_ok=True)
+        return self.root
+
+    def write_file(self, path: str, content: str) -> SandboxResult:
+        """Write content to a file under sandbox root. Creates parent dirs."""
+        try:
+            p = (self.root / path).resolve()
+            if not str(p).startswith(str(self.root)):
+                return SandboxResult(
+                    success=False,
+                    error="Path escapes sandbox root",
+                )
+            p.parent.mkdir(parents=True, exist_ok=True)
+            p.write_text(content, encoding="utf-8")
+            return SandboxResult(success=True, stdout=f"Wrote {len(content)} chars to {path}")
+        except Exception as e:
+            return SandboxResult(success=False, error=str(e))
+
+    def read_file(self, path: str) -> SandboxResult:
+        """Read file content under sandbox root."""
+        try:
+            p = (self.root / path).resolve()
+            if not str(p).startswith(str(self.root)) or not p.is_file():
+                return SandboxResult(success=False, error=f"File not in sandbox or missing: {path}")
+            content = p.read_text(encoding="utf-8", errors="replace")
+            return SandboxResult(success=True, stdout=content)
+        except Exception as e:
+            return SandboxResult(success=False, error=str(e))
+
+    def run(
+        self,
+        command: str | list[str],
+        cwd: str | Path | None = None,
+        env: dict | None = None,
+        timeout_seconds: int | None = None,
+    ) -> SandboxResult:
+        """
+        Run a shell command in the sandbox with timeout and optional resource limits.
+        """
+        timeout = timeout_seconds or self.limits.timeout_seconds
+        work_dir = Path(cwd) if cwd else self._cwd()
+        if not work_dir.exists():
+            work_dir.mkdir(parents=True, exist_ok=True)
+        full_env = os.environ.copy()
+        if env:
+            full_env.update(env)
+
+        if isinstance(command, str):
+            cmd_list = shlex.split(command)
+        else:
+            cmd_list = list(command)
+
+        def _set_limits() -> None:
+            if resource is None:
+                return
+            if self.limits.max_memory_mb is not None:
+                try:
+                    resource.setrlimit(
+                        resource.RLIMIT_AS,
+                        (self.limits.max_memory_mb * 1024 * 1024, -1),
+                    )
+                except (ValueError, OSError):
+                    pass
+            if self.limits.max_cpu_seconds is not None:
+                try:
+                    resource.setrlimit(
+                        resource.RLIMIT_CPU,
+                        (self.limits.max_cpu_seconds, -1),
+                    )
+                except (ValueError, OSError):
+                    pass
+
+        try:
+            proc = subprocess.run(
+                cmd_list,
+                cwd=str(work_dir),
+                env=full_env,
+                capture_output=True,
+                text=True,
+                timeout=timeout,
+                preexec_fn=_set_limits if os.name != "nt" else None,
+            )
+            out = (proc.stdout or "") + (
+                "\n--- stderr ---\n" + (proc.stderr or "") if proc.stderr else ""
+            )
+            return SandboxResult(
+                success=proc.returncode == 0,
+                stdout=out.strip(),
+                stderr=proc.stderr or "",
+                returncode=proc.returncode,
+            )
+        except subprocess.TimeoutExpired:
+            return SandboxResult(
+                success=False,
+                timed_out=True,
+                error=f"Command timed out after {timeout}s",
+            )
+        except Exception as e:
+            return SandboxResult(success=False, error=str(e))
+
+    def install_dependencies(self, backend: bool = True, frontend: bool = False) -> SandboxResult:
+        """Install backend (pip) and optionally frontend (npm) dependencies."""
+        results: list[str] = []
+        if backend:
+            req = self.root / "backend" / "requirements.txt"
+            if not req.exists():
+                req = self.root / "requirements.txt"
+            if req.exists():
+                # Prefer uv when available (e.g. venvs without pip); else pip
+                r = self.run(
+                    ["uv", "pip", "install", "-r", str(req)],
+                    timeout_seconds=120,
+                )
+                if not r.success and r.error:
+                    python = os.environ.get("PYTHON") or sys.executable or "python3"
+                    r = self.run(
+                        [python, "-m", "pip", "install", "-r", str(req)],
+                        timeout_seconds=120,
+                    )
+                results.append(f"pip: success={r.success}" + (f" {r.error}" if r.error else ""))
+        if frontend:
+            pkg = self.root / "frontend" / "package.json"
+            if pkg.exists():
+                r = self.run("npm install", cwd=str(self.root / "frontend"), timeout_seconds=120)
+                results.append(f"npm: success={r.success}" + (f" {r.error}" if r.error else ""))
+        return SandboxResult(
+            success=all("success=True" in s for s in results) or not results,
+            stdout="\n".join(results) or "No dependency files found",
+        )
+
+    def run_tests(self, path: str = "tests") -> SandboxResult:
+        """Run tests (pytest for backend, or path to test dir)."""
+        test_dir = self.root / path
+        if not test_dir.exists():
+            test_dir = self.root / "backend" / "tests"
+        if not test_dir.exists():
+            test_dir = self.root
+        env = os.environ.copy()
+        backend_dir = self.root / "backend"
+        if backend_dir.exists():
+            env["PYTHONPATH"] = str(backend_dir) + os.pathsep + env.get("PYTHONPATH", "")
+        python = os.environ.get("PYTHON") or sys.executable or "python3"
+        cmd = [
+            python,
+            "-m",
+            "pytest",
+            str(test_dir),
+            "-v",
+            "--tb=short",
+        ]
+        return self.run(cmd, timeout_seconds=60, env=env)