devobin 1.0.0__py3-none-any.whl

devobin/core/tools.py

@@ -0,0 +1,283 @@
+"""Agent tools — autonomous file discovery, search, and reading over a workspace.
+
+These are the primitives an agent (or DevObin's own scanner) uses to explore a
+project the way Claude Code / MimoCode do: find files by pattern, search their
+contents, read them without artificial line caps, and list directories. Every
+operation is READ-ONLY — DevObin never edits the user's project.
+
+The model outputs tool calls as [TOOL:name:argument] tags in its response.
+The chat loop detects them, executes the tools, feeds results back, and the
+model continues until it produces a final response without tool calls.
+"""
+
+from __future__ import annotations
+
+import fnmatch
+import re
+from dataclasses import dataclass, field
+from pathlib import Path
+
+from devobin.core.scanner import BINARY_EXTS, IGNORE_DIRS
+
+# ── Tool call tag format ──────────────────────────────────────────
+# [TOOL:glob:**/*.py]
+# [TOOL:grep:class\s+\w+Provider]
+# [TOOL:read:devobin/main.py]
+# [TOOL:ls:devobin/core/]
+
+TOOL_TAG_RE = re.compile(r"\[TOOL:(\w+):([^\]]+)\]")
+
+
+@dataclass
+class ToolCall:
+    """A parsed tool call from the model's output."""
+
+    tool: str
+    arg: str
+
+
+@dataclass
+class ToolResult:
+    """Uniform result for every tool call."""
+
+    tool: str
+    ok: bool = True
+    summary: str = ""
+    matches: list[str] = field(default_factory=list)
+    content: str = ""
+    error: str = ""
+
+    def render(self) -> str:
+        if not self.ok:
+            return f"[{self.tool}] error: {self.error}"
+        if self.content:
+            return self.content
+        if self.matches:
+            head = f"[{self.tool}] {len(self.matches)} match(es):\n"
+            return head + "\n".join(self.matches)
+        return self.summary or f"[{self.tool}] no results"
+
+
+def parse_tool_calls(text: str) -> list[ToolCall]:
+    """Extract all [TOOL:name:arg] tags from the model's output."""
+    return [ToolCall(tool=m.group(1), arg=m.group(2)) for m in TOOL_TAG_RE.finditer(text)]
+
+
+def strip_tool_calls(text: str) -> str:
+    """Remove all [TOOL:...] tags from the output, leaving clean text."""
+    return TOOL_TAG_RE.sub("", text).strip()
+
+
+def execute_tool_calls(calls: list[ToolCall]) -> str:
+    """Execute a list of tool calls and return formatted results."""
+    if not calls:
+        return ""
+    parts: list[str] = []
+    for call in calls:
+        fn = TOOLS.get(call.tool)
+        if fn is None:
+            parts.append(f"[{call.tool}] unknown tool. Available: {', '.join(TOOLS)}")
+            continue
+        try:
+            result: ToolResult = fn(call.arg)
+            parts.append(result.render())
+        except Exception as e:
+            parts.append(f"[{call.tool}] error: {e}")
+    return "\n\n---\n\n".join(parts)
+
+
+def _iter_files(root: Path):
+    """Yield text-ish files under root, skipping ignored dirs and binaries."""
+    for path in root.rglob("*"):
+        if not path.is_file():
+            continue
+        rel_parts = path.relative_to(root).parts
+        if any(p in IGNORE_DIRS for p in rel_parts):
+            continue
+        if path.suffix.lower() in BINARY_EXTS:
+            continue
+        yield path
+
+
+def glob_files(pattern: str, root: str | Path | None = None, limit: int = 500) -> ToolResult:
+    """Find files whose relative path matches a glob pattern (e.g. ``**/*.py``)."""
+    root_path = Path(root).resolve() if root else Path.cwd()
+    matches: list[str] = []
+    for path in _iter_files(root_path):
+        rel = path.relative_to(root_path).as_posix()
+        if fnmatch.fnmatch(rel, pattern) or fnmatch.fnmatch(path.name, pattern):
+            matches.append(rel)
+            if len(matches) >= limit:
+                break
+    return ToolResult(tool="glob", matches=sorted(matches),
+                      summary=f"{len(matches)} file(s) matching '{pattern}'")
+
+
+def grep_files(
+    pattern: str,
+    root: str | Path | None = None,
+    glob: str = "*",
+    limit: int = 300,
+    ignore_case: bool = True,
+) -> ToolResult:
+    """Search file contents with a regex, returning ``path:line: text`` hits."""
+    root_path = Path(root).resolve() if root else Path.cwd()
+    try:
+        rx = re.compile(pattern, re.IGNORECASE if ignore_case else 0)
+    except re.error as e:
+        return ToolResult(tool="grep", ok=False, error=f"bad regex: {e}")
+
+    hits: list[str] = []
+    for path in _iter_files(root_path):
+        rel = path.relative_to(root_path).as_posix()
+        if not (fnmatch.fnmatch(rel, glob) or fnmatch.fnmatch(path.name, glob)):
+            continue
+        try:
+            for i, line in enumerate(path.read_text(encoding="utf-8", errors="replace").splitlines(), 1):
+                if rx.search(line):
+                    hits.append(f"{rel}:{i}: {line.strip()[:200]}")
+                    if len(hits) >= limit:
+                        return ToolResult(tool="grep", matches=hits,
+                                          summary=f"{len(hits)}+ hits (truncated)")
+        except Exception:
+            continue
+    return ToolResult(tool="grep", matches=hits, summary=f"{len(hits)} hit(s) for '{pattern}'")
+
+
+def read_file(path: str, root: str | Path | None = None, max_bytes: int = 2_000_000) -> ToolResult:
+    """Read a file with a generous cap (no 500-line truncation)."""
+    root_path = Path(root).resolve() if root else Path.cwd()
+    target = (root_path / path).resolve() if not Path(path).is_absolute() else Path(path).resolve()
+
+    if not target.exists() or not target.is_file():
+        return ToolResult(tool="read", ok=False, error=f"not a file: {path}")
+    if target.suffix.lower() in BINARY_EXTS:
+        return ToolResult(tool="read", ok=False, error=f"binary file skipped: {path}")
+
+    try:
+        data = target.read_bytes()[:max_bytes]
+        text = data.decode("utf-8", errors="replace")
+    except Exception as e:
+        return ToolResult(tool="read", ok=False, error=str(e))
+
+    lines = text.count("\n") + 1
+    return ToolResult(tool="read", content=text,
+                      summary=f"{path} ({lines} lines, {len(data)} bytes)")
+
+
+def list_dir(path: str = ".", root: str | Path | None = None) -> ToolResult:
+    """List a directory's entries (dirs marked with a trailing slash)."""
+    root_path = Path(root).resolve() if root else Path.cwd()
+    target = (root_path / path).resolve()
+    if not target.exists() or not target.is_dir():
+        return ToolResult(tool="ls", ok=False, error=f"not a directory: {path}")
+    try:
+        entries = sorted(
+            (e.name + "/" if e.is_dir() else e.name)
+            for e in target.iterdir()
+            if e.name not in IGNORE_DIRS
+        )
+    except Exception as e:
+        return ToolResult(tool="ls", ok=False, error=str(e))
+    return ToolResult(tool="ls", matches=entries, summary=f"{len(entries)} entr(ies) in {path}")
+
+
+def summarize_results(calls: list[ToolCall], raw_results: str) -> str:
+    """Create a user-friendly summary of tool results (no full code shown).
+
+    The full content is available to the model via the conversation history,
+    but the user only sees file names, line counts, and key identifiers.
+    """
+    parts: list[str] = []
+    for call in calls:
+        if call.tool == "read":
+            # Extract just the filename and stats from the result
+            lines = raw_results.split("\n")
+            # Find the relevant section for this tool call
+            fname = call.arg
+            line_count = 0
+            classes = []
+            functions = []
+            for line in lines:
+                stripped = line.strip()
+                if stripped.startswith("class ") and "(" in stripped:
+                    name = stripped.split("(")[0].replace("class ", "").strip()
+                    classes.append(name)
+                elif stripped.startswith("def ") and "(" in stripped:
+                    name = stripped.split("(")[0].replace("def ", "").strip()
+                    if not name.startswith("_"):
+                        functions.append(name)
+                elif stripped and not stripped.startswith("["):
+                    line_count += 1
+            summary = f"  {fname} — {line_count} lines"
+            if classes:
+                summary += f" | classes: {', '.join(classes[:8])}"
+            if functions:
+                summary += f" | functions: {', '.join(functions[:8])}"
+            parts.append(summary)
+        elif call.tool == "glob":
+            # Show file count and first few matches
+            match_lines = [l for l in raw_results.split("\n") if l.strip() and not l.startswith("[")]
+            parts.append(f"  Found {len(match_lines)} files matching '{call.arg}'")
+        elif call.tool == "grep":
+            match_lines = [l for l in raw_results.split("\n") if l.strip() and ":" in l and not l.startswith("[")]
+            parts.append(f"  {len(match_lines)} matches for '{call.arg}'")
+        elif call.tool == "ls":
+            parts.append(f"  Directory '{call.arg}':")
+            for line in raw_results.split("\n"):
+                if line.strip() and not line.startswith("["):
+                    parts.append(f"    {line.strip()}")
+    return "\n".join(parts) if parts else raw_results[:500]
+
+
+def write_file(path: str, content: str, root: str | Path | None = None) -> ToolResult:
+    """Write a .md prompt file to the working directory.
+
+    RESTRICTIONS:
+    - Only .md files are allowed (this is a prompt tool, not a code editor)
+    - Path must be relative (no absolute paths)
+    - No directory traversal (no ..)
+    - Only writes to the current working directory
+    """
+    root_path = Path(root).resolve() if root else Path.cwd()
+
+    # Security: only .md files
+    if not path.endswith(".md"):
+        return ToolResult(tool="write", ok=False,
+                          error="Only .md files are allowed. This is a prompt tool.")
+
+    # Security: no absolute paths
+    if Path(path).is_absolute():
+        return ToolResult(tool="write", ok=False,
+                          error="Only relative paths allowed. Use a filename like 'my_prompt.md'.")
+
+    # Security: no directory traversal
+    if ".." in path:
+        return ToolResult(tool="write", ok=False,
+                          error="Directory traversal not allowed.")
+
+    target = (root_path / path).resolve()
+
+    # Security: must be within the working directory
+    if not str(target).startswith(str(root_path)):
+        return ToolResult(tool="write", ok=False,
+                          error="Cannot write outside the working directory.")
+
+    try:
+        target.parent.mkdir(parents=True, exist_ok=True)
+        target.write_text(content, encoding="utf-8")
+        lines = content.count("\n") + 1
+        return ToolResult(tool="write", ok=True,
+                          summary=f"Created: {path} ({lines} lines, {len(content)} bytes)")
+    except Exception as e:
+        return ToolResult(tool="write", ok=False, error=str(e))
+
+
+# Registry so a model/agent loop can dispatch by name.
+TOOLS = {
+    "glob": glob_files,
+    "grep": grep_files,
+    "read": read_file,
+    "ls": list_dir,
+    "write": write_file,
+}

devobin/core/validator.py

@@ -0,0 +1,235 @@
+"""Output validator — checks code length AND grounding in the real project.
+
+DevObin is a Prompt Architect: it may show SHORT snippets to sharpen a prompt,
+but must never deliver complete files or working implementations. It must also
+never hallucinate technologies that don't exist in the scanned workspace.
+"""
+
+from __future__ import annotations
+
+import re
+from dataclasses import dataclass, field
+
+
+@dataclass
+class ValidationResult:
+    """Result of validating AI output."""
+
+    is_clean: bool
+    code_score: int
+    detected_patterns: list[str]
+    reason: str = ""
+
+    @property
+    def summary(self) -> str:
+        if self.is_clean:
+            return "OK"
+        return f"Full code detected (score={self.code_score}): {', '.join(self.detected_patterns[:5])}"
+
+
+@dataclass
+class GroundingResult:
+    """Result of checking if the prompt is grounded in the real project."""
+
+    is_grounded: bool
+    hallucinations: list[str] = field(default_factory=list)
+    warnings: list[str] = field(default_factory=list)
+
+    @property
+    def summary(self) -> str:
+        if self.is_grounded:
+            return "OK"
+        return f"Hallucinated: {', '.join(self.hallucinations[:5])}"
+
+
+# ── Code length policy ────────────────────────────────────────────
+MAX_SNIPPET_LINES = 12
+MAX_SNIPPET_BLOCKS = 4
+MAX_STRUCTURAL_DEFS = 3
+THRESHOLD = 10
+
+# ── Full-document markers (never in an illustrative snippet) ──────
+FULL_DOC_PATTERNS: list[tuple[str, str]] = [
+    (r"<!DOCTYPE\s+html", "HTML document"),
+    (r"<html[\s>]", "full <html> tree"),
+    (r"<head\s*>", "<head> section"),
+    (r"<body[\s>]", "<body> section"),
+]
+
+STRUCTURAL_PATTERNS: list[tuple[str, str]] = [
+    (r"^\s*def\s+\w+\s*\(", "python def"),
+    (r"^\s*class\s+\w+\s*[\(:]", "python class"),
+    (r"^\s*(export\s+)?(async\s+)?function\s+\w+\s*\(", "js function"),
+]
+
+_FENCE_RE = re.compile(r"```[^\n]*\n(.*?)```", re.DOTALL)
+
+# ── Technologies that models commonly hallucinate ──────────────────
+# These are only "hallucinated" if the workspace has NO evidence of them.
+TECH_KEYWORDS: dict[str, list[str]] = {
+    "PostgreSQL": ["postgresql", "postgres", "psql", "asyncpg", "sqlalchemy", r"models\.py", r"migrations/"],
+    "MySQL": ["mysql", "mariadb", "pymysql"],
+    "Docker": ["dockerfile", "docker-compose", "docker"],
+    "React": ["react", "jsx", "tsx", r"next\.js", "nextjs"],
+    "Vue": ["vue", "vuejs", "nuxt"],
+    "Angular": ["angular"],
+    "Bootstrap": ["bootstrap"],
+    "Tailwind": ["tailwind"],
+    "Redis": ["redis"],
+    "Kubernetes": ["kubernetes", "k8s"],
+    "GraphQL": ["graphql"],
+    "Celery": ["celery"],
+    "RabbitMQ": ["rabbitmq"],
+    "Kafka": ["kafka"],
+}
+
+
+def _code_blocks(text: str) -> list[str]:
+    return _FENCE_RE.findall(text)
+
+
+def _code_line_count(block: str) -> int:
+    return sum(1 for line in block.splitlines() if line.strip())
+
+
+def validate_output(text: str) -> ValidationResult:
+    """Validate AI output for code length (short snippets OK, full files blocked)."""
+    if not text or not text.strip():
+        return ValidationResult(is_clean=True, code_score=0, detected_patterns=[])
+
+    violations: list[str] = []
+    score = 0
+
+    for pattern, label in FULL_DOC_PATTERNS:
+        if re.search(pattern, text, re.MULTILINE | re.IGNORECASE):
+            violations.append(label)
+            score += 15
+
+    blocks = _code_blocks(text)
+    for block in blocks:
+        n = _code_line_count(block)
+        if n > MAX_SNIPPET_LINES:
+            violations.append(f"{n}-line code block (max {MAX_SNIPPET_LINES})")
+            score += 15
+
+    if len(blocks) > MAX_SNIPPET_BLOCKS:
+        violations.append(f"{len(blocks)} code blocks (max {MAX_SNIPPET_BLOCKS})")
+        score += 10
+
+    struct_hits = 0
+    for pattern, _label in STRUCTURAL_PATTERNS:
+        struct_hits += len(re.findall(pattern, text, re.MULTILINE))
+    if struct_hits > MAX_STRUCTURAL_DEFS:
+        violations.append(f"{struct_hits} code definitions (max {MAX_STRUCTURAL_DEFS})")
+        score += 10
+
+    is_clean = score < THRESHOLD
+    reason = "" if is_clean else f"Full code detected (score={score})"
+
+    return ValidationResult(
+        is_clean=is_clean,
+        code_score=score,
+        detected_patterns=violations,
+        reason=reason,
+    )
+
+
+def validate_grounding(text: str, project_map=None) -> GroundingResult:
+    """Check if the generated prompt references technologies actually in the project.
+
+    This catches the common failure mode where the model hallucinates PostgreSQL,
+    Docker, React, etc. despite the project having none of those.
+    """
+    if not text or not text.strip():
+        return GroundingResult(is_grounded=True)
+
+    text_lower = text.lower()
+    hallucinations: list[str] = []
+    warnings: list[str] = []
+
+    # Build a set of evidence strings from the workspace scan
+    # Only count source code and config files, NOT documentation (README etc.)
+    evidence = set()
+    if project_map is not None:
+        # Add file extensions and names (but skip docs)
+        for f in getattr(project_map, "files", []):
+            fname_lower = f.name.lower()
+            if fname_lower.startswith("readme") or fname_lower.endswith(".md"):
+                continue  # Skip documentation files
+            evidence.add(f.ext.lower())
+            evidence.add(fname_lower)
+            evidence.add(f.path.lower())
+        # Add directory names
+        for d in getattr(project_map, "dirs", []):
+            evidence.add(d.lower())
+        # Add manifest content — only config files, not docs
+        for rel, content in getattr(project_map, "manifests", {}).items():
+            rel_lower = rel.lower()
+            if "readme" in rel_lower or rel_lower.endswith(".md"):
+                continue  # Skip documentation
+            evidence.add(content.lower())
+        # Add languages
+        for lang in getattr(project_map, "languages", {}).keys():
+            evidence.add(lang.lower())
+
+    evidence_text = " ".join(evidence)
+
+    for tech, keywords in TECH_KEYWORDS.items():
+        # Check if the prompt mentions this technology
+        mentioned = any(re.search(kw, text_lower) for kw in keywords)
+        if not mentioned:
+            continue
+
+        # Check if there's evidence of this technology in the workspace
+        has_evidence = any(re.search(kw, evidence_text) for kw in keywords)
+
+        if not has_evidence:
+            hallucinations.append(tech)
+            warnings.append(
+                f"'{tech}' is mentioned in the prompt but NOT found in the "
+                f"scanned workspace. The model may have hallucinated it."
+            )
+
+    is_grounded = len(hallucinations) == 0
+    return GroundingResult(
+        is_grounded=is_grounded,
+        hallucinations=hallucinations,
+        warnings=warnings,
+    )
+
+
+def build_rewrite_prompt(original_text: str) -> str:
+    """Build a prompt to rewrite a full implementation back into instructions."""
+    return """Your previous response included a full implementation, which breaks DevObin's rules.
+
+You are a PROMPT ARCHITECT, not a CODER:
+- You create instructions FOR other AI coding agents.
+- You do NOT deliver complete files or working applications yourself.
+
+Rewrite your previous response as an engineering prompt with:
+1. Technical specification
+2. Architecture requirements
+3. Implementation instructions
+4. Optimization rules
+5. Security rules
+
+You MAY keep at most a few SHORT illustrative snippets (max 12 lines each) purely to
+clarify an instruction — a function signature, a config key, a data schema.
+You may NOT include:
+- Complete files or full applications
+- Long or copy-paste-ready implementations
+- More than a few tiny snippets
+
+The result must be a prompt another AI coding agent can implement from."""
+
+
+def get_mode_message() -> str:
+    """Get the mode indicator for UI display."""
+    return (
+        "DevObin AI\n"
+        "Prompt Architect Mode\n"
+        "\n"
+        "Full code: Disabled\n"
+        "Short illustrative snippets: OK\n"
+        "Engineering context only"
+    )

devobin/main.py

@@ -0,0 +1,33 @@
+"""DevObin CLI application entry point."""
+
+from __future__ import annotations
+
+import typer
+from rich.console import Console
+
+from devobin import __version__, __app_name__
+
+console = Console()
+
+app = typer.Typer(
+    name="devobin",
+    help=f"{__app_name__} — AI Engineering Context Compiler & Prompt Architect",
+    add_completion=False,
+    no_args_is_help=False,
+    invoke_without_command=True,
+)
+
+
+@app.callback(invoke_without_command=True)
+def main(
+    ctx: typer.Context,
+    version: bool = typer.Option(False, "--version", "-v", help="Show version"),
+) -> None:
+    """DevObin AI — Interactive Engineering Context Compiler."""
+    if version:
+        console.print(f"{__app_name__} v{__version__}")
+        raise typer.Exit()
+
+    if ctx.invoked_subcommand is None:
+        from devobin.app import run_app
+        run_app()

devobin/output/__init__.py

@@ -0,0 +1 @@
+"""DevObin output package."""

devobin/providers/__init__.py

@@ -0,0 +1,85 @@
+"""AI provider abstraction layer."""
+
+from __future__ import annotations
+
+from abc import ABC, abstractmethod
+from typing import AsyncIterator
+
+from devobin.config.settings import get_config
+
+
+class AIProvider(ABC):
+    """Base class for all AI providers."""
+
+    name: str = "base"
+
+    def __init__(self, config: dict) -> None:
+        self.config = config
+        self.api_key = config.get("api_key", "")
+        self.model = config.get("model", "")
+        self.base_url = config.get("base_url", "")
+
+    @abstractmethod
+    async def chat(
+        self,
+        messages: list[dict[str, str]],
+        system: str | None = None,
+        temperature: float = 0.7,
+        max_tokens: int = 4096,
+    ) -> str:
+        """Send a chat completion request and return the response text."""
+        ...
+
+    @abstractmethod
+    async def chat_stream(
+        self,
+        messages: list[dict[str, str]],
+        system: str | None = None,
+        temperature: float = 0.7,
+        max_tokens: int = 4096,
+    ) -> AsyncIterator[str]:
+        """Stream chat completion tokens."""
+        ...
+        yield  # type: ignore[misc]
+
+    @abstractmethod
+    async def validate_connection(self) -> bool:
+        """Validate that the provider is properly configured and reachable."""
+        ...
+
+    @abstractmethod
+    def available_models(self) -> list[str]:
+        """Return list of available model IDs for this provider."""
+        ...
+
+
+def get_provider(name: str | None = None) -> AIProvider:
+    """Get an AI provider instance by name (or the active one)."""
+    from devobin.providers.openai_provider import OpenAIProvider
+    from devobin.providers.anthropic_provider import AnthropicProvider
+    from devobin.providers.google_provider import GoogleProvider
+    from devobin.providers.ollama_provider import OllamaProvider
+
+    config = get_config()
+    provider_name = name or config.active_provider
+    if not provider_name:
+        raise ValueError("No AI provider configured. Use /connect to set one up.")
+
+    provider_cfg = config.get_provider(provider_name)
+    if not provider_cfg:
+        raise ValueError(f"Provider '{provider_name}' not found. Use /connect to configure.")
+
+    registry: dict[str, type[AIProvider]] = {
+        "openai": OpenAIProvider,
+        "anthropic": AnthropicProvider,
+        "google": GoogleProvider,
+        "ollama": OllamaProvider,
+        "openai-compatible": OpenAIProvider,
+    }
+
+    provider_type = provider_cfg.get("type", provider_name)
+    cls = registry.get(provider_type)
+    if cls is None:
+        raise ValueError(f"Unknown provider type: {provider_type}")
+
+    return cls(provider_cfg)